chore(release): 0.7.0

Reviewed-on: #30

CHANGELOG.md

@@ -2,6 +2,38 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [0.7.0](https://gitea.deepak.science:2222/physics/deepdog/compare/0.6.7...0.7.0) (2023-05-01)
+
+
+### ⚠ BREAKING CHANGES
+
+* removes fastfilter parameter because it should never be needed
+
+### Features
+
+* adds pair capability to real spectrum run hopefully ([a089951](https://gitea.deepak.science:2222/physics/deepdog/commit/a089951bbefcd8a0b2efeb49b7a8090412cbb23d))
+* removes fastfilter parameter because it should never be needed ([a015daf](https://gitea.deepak.science:2222/physics/deepdog/commit/a015daf5ff6fa5f6155c8d7e02981b588840a5b0))
+
+### [0.6.7](https://gitea.deepak.science:2222/physics/deepdog/compare/0.6.6...0.6.7) (2023-04-14)
+
+
+### Features
+
+* adds option to cap core count for real spectrum run ([bf15f4a](https://gitea.deepak.science:2222/physics/deepdog/commit/bf15f4a7b7f59504983624e7d512ed7474372032))
+* adds option to cap core count for temp aware run ([12903b2](https://gitea.deepak.science:2222/physics/deepdog/commit/12903b2540cefb040174d230bc0d04719a6dc1b7))
+
+
+### Bug Fixes
+
+* avoids redefinition of core count in loop ([1cf4454](https://gitea.deepak.science:2222/physics/deepdog/commit/1cf44541531541088198bd4599d467df3e1acbcf))
+
+### [0.6.6](https://gitea.deepak.science:2222/physics/deepdog/compare/0.6.5...0.6.6) (2023-04-09)
+
+
+### Bug Fixes
+
+* removes bad logging in multiprocessing function ([8fd1b75](https://gitea.deepak.science:2222/physics/deepdog/commit/8fd1b75e1378301210bfa8f14dd09174bbd21414))
+
 ### [0.6.5](https://gitea.deepak.science:2222/physics/deepdog/compare/0.6.4...0.6.5) (2023-04-09)
 
 

deepdog/real_spectrum_run.py

@@ -5,7 +5,7 @@ import pdme.measurement.input_types
 import pdme.measurement.oscillating_dipole
 import pdme.util.fast_v_calc
 import pdme.util.fast_nonlocal_spectrum
-from typing import Sequence, Tuple, List, Dict, Union
+from typing import Sequence, Tuple, List, Dict, Union, Optional
 import datetime
 import csv
 import multiprocessing
@@ -20,16 +20,50 @@ CHUNKSIZE = 50
 _logger = logging.getLogger(__name__)
 
 
-def get_a_result(input) -> int:
-	model, dot_inputs, lows, highs, monte_carlo_count, seed = input
+def get_a_result_fast_filter_pairs(input) -> int:
+	(
+		model,
+		dot_inputs,
+		lows,
+		highs,
+		pair_inputs,
+		pair_lows,
+		pair_highs,
+		monte_carlo_count,
+		seed,
+	) = input
 
 	rng = numpy.random.default_rng(seed)
 	# TODO: A long term refactor is to pull the frequency stuff out from here. The None stands for max_frequency, which is unneeded in the actually useful models.
 	sample_dipoles = model.get_monte_carlo_dipole_inputs(
 		monte_carlo_count, None, rng_to_use=rng
 	)
-	vals = pdme.util.fast_v_calc.fast_vs_for_dipoleses(dot_inputs, sample_dipoles)
-	return numpy.count_nonzero(pdme.util.fast_v_calc.between(vals, lows, highs))
+
+	current_sample = sample_dipoles
+	for di, low, high in zip(dot_inputs, lows, highs):
+
+		if len(current_sample) < 1:
+			break
+		vals = pdme.util.fast_v_calc.fast_vs_for_dipoleses(
+			numpy.array([di]), current_sample
+		)
+
+		current_sample = current_sample[numpy.all((vals > low) & (vals < high), axis=1)]
+
+	for pi, plow, phigh in zip(pair_inputs, pair_lows, pair_highs):
+		if len(current_sample) < 1:
+			break
+		vals = pdme.util.fast_nonlocal_spectrum.fast_s_nonlocal_dipoleses(
+			numpy.array([pi]), current_sample
+		)
+
+		current_sample = current_sample[
+			numpy.all(
+				((vals > plow) & (vals < phigh)) | ((vals < plow) & (vals > phigh)),
+				axis=1,
+			)
+		]
+	return len(current_sample)
 
 
 def get_a_result_fast_filter(input) -> int:
@@ -87,7 +121,10 @@ class RealSpectrumRun:
 		max_monte_carlo_cycles_steps: int = 10,
 		chunksize: int = CHUNKSIZE,
 		initial_seed: int = 12345,
-		use_fast_filter: bool = True,
+		cap_core_count: int = 0,
+		pair_measurements: Optional[
+			Sequence[pdme.measurement.DotPairRangeMeasurement]
+		] = None,
 	) -> None:
 		self.measurements = measurements
 		self.dot_inputs = [(measure.r, measure.f) for measure in self.measurements]
@@ -96,6 +133,21 @@ class RealSpectrumRun:
 			self.dot_inputs
 		)
 
+		if pair_measurements is not None:
+			self.pair_measurements = pair_measurements
+			self.use_pair_measurements = True
+			self.dot_pair_inputs = [
+				(measure.r1, measure.r2, measure.f)
+				for measure in self.pair_measurements
+			]
+			self.dot_pair_inputs_array = (
+				pdme.measurement.input_types.dot_pair_inputs_to_array(
+					self.dot_pair_inputs
+				)
+			)
+		else:
+			self.use_pair_measurements = False
+
 		self.models = [model for (_, model) in models_with_names]
 		self.model_names = [name for (name, _) in models_with_names]
 		self.model_count = len(self.models)
@@ -116,13 +168,14 @@ class RealSpectrumRun:
 		self.probabilities = [1 / self.model_count] * self.model_count
 
 		timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
-		self.use_fast_filter = use_fast_filter
-		ff_string = "no_fast_filter"
-		if self.use_fast_filter:
-			ff_string = "fast_filter"
+
+		ff_string = "fast_filter"
+
 		self.filename = f"{timestamp}-{filename_slug}.realdata.{ff_string}.bayesrun.csv"
 		self.initial_seed = initial_seed
 
+		self.cap_core_count = cap_core_count
+
 	def go(self) -> None:
 		with open(self.filename, "a", newline="") as outfile:
 			writer = csv.DictWriter(outfile, fieldnames=self.csv_fields, dialect="unix")
@@ -135,16 +188,29 @@ class RealSpectrumRun:
 			self.measurements
 		)
 
+		pair_lows = None
+		pair_highs = None
+		if self.use_pair_measurements:
+			(
+				pair_lows,
+				pair_highs,
+			) = pdme.measurement.input_types.dot_range_measurements_low_high_arrays(
+				self.pair_measurements
+			)
+
 		# define a new seed sequence for each run
 		seed_sequence = numpy.random.SeedSequence(self.initial_seed)
 
 		results = []
 		_logger.debug("Going to iterate over models now")
+		core_count = multiprocessing.cpu_count() - 1 or 1
+		if (self.cap_core_count >= 1) and (self.cap_core_count < core_count):
+			core_count = self.cap_core_count
+		_logger.info(f"Using {core_count} cores")
 		for model_count, (model, model_name) in enumerate(
 			zip(self.models, self.model_names)
 		):
 			_logger.debug(f"Doing model #{model_count}: {model_name}")
-			core_count = multiprocessing.cpu_count() - 1 or 1
 			with multiprocessing.Pool(core_count) as pool:
 				cycle_count = 0
 				cycle_success = 0
@@ -162,27 +228,46 @@ class RealSpectrumRun:
 					# that way we get more stuff.
 					seeds = seed_sequence.spawn(self.monte_carlo_cycles)
 
-					if self.use_fast_filter:
-						result_func = get_a_result_fast_filter
-					else:
-						result_func = get_a_result
-					current_success = sum(
-						pool.imap_unordered(
-							result_func,
-							[
-								(
-									model,
-									self.dot_inputs_array,
-									lows,
-									highs,
-									self.monte_carlo_count,
-									seed,
-								)
-								for seed in seeds
-							],
-							self.chunksize,
+					if self.use_pair_measurements:
+						current_success = sum(
+							pool.imap_unordered(
+								get_a_result_fast_filter_pairs,
+								[
+									(
+										model,
+										self.dot_inputs_array,
+										lows,
+										highs,
+										self.dot_pair_inputs_array,
+										pair_lows,
+										pair_highs,
+										self.monte_carlo_count,
+										seed,
+									)
+									for seed in seeds
+								],
+								self.chunksize,
+							)
+						)
+					else:
+
+						current_success = sum(
+							pool.imap_unordered(
+								get_a_result_fast_filter,
+								[
+									(
+										model,
+										self.dot_inputs_array,
+										lows,
+										highs,
+										self.monte_carlo_count,
+										seed,
+									)
+									for seed in seeds
+								],
+								self.chunksize,
+							)
 						)
-					)
 
 					cycle_success += current_success
 					_logger.debug(f"current running successes: {cycle_success}")

deepdog/temp_aware_real_spectrum_run.py

@@ -40,8 +40,6 @@ def get_a_result_fast_filter(input) -> int:
 	for temp in dot_inputs_dict.keys():
 		dot_inputs = dot_inputs_dict[temp]
 		lows, highs = low_high_dict[temp]
-		_logger.info(f"current sample length: {len(current_sample)}")
-		_logger.info(f"workng on temp {temp}")
 		for di, low, high in zip(dot_inputs, lows, highs):
 
 			if len(current_sample) < 1:
@@ -92,6 +90,7 @@ class TempAwareRealSpectrumRun:
 		max_monte_carlo_cycles_steps: int = 10,
 		chunksize: int = CHUNKSIZE,
 		initial_seed: int = 12345,
+		cap_core_count: int = 0,
 	) -> None:
 		self.measurements_dict = measurements_dict
 		self.dot_inputs_dict = {
@@ -128,6 +127,8 @@ class TempAwareRealSpectrumRun:
 		self.filename = f"{timestamp}-{filename_slug}.realdata.{ff_string}.bayesrun.csv"
 		self.initial_seed = initial_seed
 
+		self.cap_core_count = cap_core_count
+
 	def go(self) -> None:
 		with open(self.filename, "a", newline="") as outfile:
 			writer = csv.DictWriter(outfile, fieldnames=self.csv_fields, dialect="unix")
@@ -148,11 +149,14 @@ class TempAwareRealSpectrumRun:
 
 		results = []
 		_logger.debug("Going to iterate over models now")
+		core_count = multiprocessing.cpu_count() - 1 or 1
+		if (self.cap_core_count >= 1) and (self.cap_core_count < core_count):
+			core_count = self.cap_core_count
+		_logger.info(f"Using {core_count} cores")
 		for model_count, (model, model_name) in enumerate(
 			zip(self.models, self.model_names)
 		):
 			_logger.debug(f"Doing model #{model_count}: {model_name}")
-			core_count = multiprocessing.cpu_count() - 1 or 1
 			with multiprocessing.Pool(core_count) as pool:
 				cycle_count = 0
 				cycle_success = 0

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "deepdog"
-version = "0.6.5"
+version = "0.7.0"
 description = ""
 authors = ["Deepak Mallubhotla <dmallubhotla+github@gmail.com>"]