chore(release): 0.7.9

Reviewed-on: #37

CHANGELOG.md

@@ -2,6 +2,49 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [0.7.9](https://gitea.deepak.science:2222/physics/deepdog/compare/0.7.8...0.7.9) (2024-04-21)
+
+
+### Features
+
+* adds ability to write custom dmc filters ([ea080ca](https://gitea.deepak.science:2222/physics/deepdog/commit/ea080ca1c7068042ce1e0a222d317f785a6b05f4))
+* adds tarucha phase calculation, using spin qubit precession rate noise ([3ae0783](https://gitea.deepak.science:2222/physics/deepdog/commit/3ae0783d00cbe6a76439c1d671f2cff621d8d0a8))
+
+### [0.7.8](https://gitea.deepak.science:2222/physics/deepdog/compare/0.7.7...0.7.8) (2024-02-29)
+
+
+### Bug Fixes
+
+* uses correct measurements ([5f534a6](https://gitea.deepak.science:2222/physics/deepdog/commit/5f534a60cc7c4838fcacee11a7e58b97d34e154a))
+
+### [0.7.7](https://gitea.deepak.science:2222/physics/deepdog/compare/0.7.6...0.7.7) (2024-02-29)
+
+
+### Bug Fixes
+
+* fixes phase calculation issue with setting input array ([48e41cb](https://gitea.deepak.science:2222/physics/deepdog/commit/48e41cbd2c58d4c4d2747822d618d7d55257643d))
+
+### [0.7.6](https://gitea.deepak.science:2222/physics/deepdog/compare/0.7.5...0.7.6) (2024-02-28)
+
+
+### Features
+
+* adds ability to use phase measurements only for correlations ([bb72e90](https://gitea.deepak.science:2222/physics/deepdog/commit/bb72e903d14704a3783daf2dbc1797b90880aa85))
+
+
+### Bug Fixes
+
+* fixes typeerror vs indexerror on bare float as cost in subset simulation ([65e1948](https://gitea.deepak.science:2222/physics/deepdog/commit/65e19488359d7f5656660da7da8f32ed474989c3))
+
+### [0.7.5](https://gitea.deepak.science:2222/physics/deepdog/compare/0.7.4...0.7.5) (2023-12-09)
+
+
+### Features
+
+* adds direct monte carlo package ([1741807](https://gitea.deepak.science:2222/physics/deepdog/commit/1741807be43d08fb51bc94518dd3b67585c04c20))
+* adds longchain logging if logging last generation ([b4e5f53](https://gitea.deepak.science:2222/physics/deepdog/commit/b4e5f5372682fc64c3734a96c4a899e018f127ce))
+* allows disabling timestamp in subset simulation bayes results ([9a4548d](https://gitea.deepak.science:2222/physics/deepdog/commit/9a4548def45a01f1f518135d4237c3dc09dcc342))
+
 ### [0.7.4](https://gitea.deepak.science:2222/physics/deepdog/compare/0.7.3...0.7.4) (2023-07-27)
 
 

deepdog/bayes_run_with_ss.py

@@ -73,6 +73,7 @@ class BayesRunWithSubspaceSimulation:
 		ss_dump_last_generation=False,
 		ss_initial_costs_chunk_size=100,
 		write_output_to_bayesruncsv=True,
+		use_timestamp_for_output=True,
 	) -> None:
 		self.dot_inputs = pdme.inputs.inputs_with_frequency_range(
 			dot_positions, frequency_range
@@ -110,8 +111,11 @@ class BayesRunWithSubspaceSimulation:
 
 		self.probabilities = [1 / self.model_count] * self.model_count
 
+		if use_timestamp_for_output:
 			timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
 			self.filename = f"{timestamp}-{filename_slug}.bayesrunwithss.csv"
+		else:
+			self.filename = f"{filename_slug}.bayesrunwithss.csv"
 		self.max_frequency = max_frequency
 
 		if end_threshold is not None:

deepdog/direct_monte_carlo/__init__.py

@@ -0,0 +1,6 @@
+from deepdog.direct_monte_carlo.direct_mc import (
+	DirectMonteCarloRun,
+	DirectMonteCarloConfig,
+)
+
+__all__ = ["DirectMonteCarloRun", "DirectMonteCarloConfig"]

deepdog/direct_monte_carlo/compose_filter.py

@@ -0,0 +1,14 @@
+from typing import Sequence
+from deepdog.direct_monte_carlo.direct_mc import DirectMonteCarloFilter
+import numpy
+
+
+class ComposedDMCFilter(DirectMonteCarloFilter):
+	def __init__(self, filters: Sequence[DirectMonteCarloFilter]):
+		self.filters = filters
+
+	def filter_samples(self, samples: numpy.ndarray) -> numpy.ndarray:
+		current_sample = samples
+		for filter in self.filters:
+			current_sample = filter.filter_samples(current_sample)
+		return current_sample

deepdog/direct_monte_carlo/direct_mc.py

@@ -0,0 +1,174 @@
+import pdme.model
+import pdme.measurement
+import pdme.measurement.input_types
+import pdme.subspace_simulation
+from typing import Tuple, Dict, NewType, Any
+from dataclasses import dataclass
+import logging
+import numpy
+import numpy.random
+import pdme.util.fast_v_calc
+
+_logger = logging.getLogger(__name__)
+
+
+@dataclass
+class DirectMonteCarloResult:
+	successes: int
+	monte_carlo_count: int
+	likelihood: float
+
+
+@dataclass
+class DirectMonteCarloConfig:
+	monte_carlo_count_per_cycle: int = 10000
+	monte_carlo_cycles: int = 10
+	target_success: int = 100
+	max_monte_carlo_cycles_steps: int = 10
+	monte_carlo_seed: int = 1234
+	write_successes_to_file: bool = False
+	tag: str = ""
+
+
+# Aliasing dict as a generic data container
+DirectMonteCarloData = NewType("DirectMonteCarloData", Dict[str, Any])
+
+
+class DirectMonteCarloFilter:
+	"""
+	Abstract class for filtering out samples matching some criteria. Initialise with data as needed,
+	then filter out samples as needed.
+	"""
+
+	def filter_samples(self, samples: numpy.ndarray) -> numpy.ndarray:
+		raise NotImplementedError
+
+
+class DirectMonteCarloRun:
+	"""
+	A single model Direct Monte Carlo run, currently implemented only using single threading.
+	An encapsulation of the steps needed for a Bayes run.
+
+	Parameters
+	----------
+	model_name_pair : Sequence[Tuple(str, pdme.model.DipoleModel)]
+	The model to evaluate, with name.
+
+	measurements: Sequence[pdme.measurement.DotRangeMeasurement]
+	The measurements as dot ranges to use as the bounds for the Monte Carlo calculation.
+
+	monte_carlo_count_per_cycle: int
+	The number of Monte Carlo iterations to use in a single cycle calculation.
+
+	monte_carlo_cycles: int
+	The number of cycles to use in each step.
+	Increasing monte_carlo_count_per_cycle increases memory usage (and runtime), while this increases runtime, allowing
+	control over memory use.
+
+	target_success: int
+	The number of successes to target before exiting early.
+	Should likely be ~100 but can go higher to.
+
+	max_monte_carlo_cycles_steps: int
+	The number of steps to use. Each step consists of monte_carlo_cycles cycles, each of which has monte_carlo_count_per_cycle iterations.
+
+	monte_carlo_seed: int
+	The seed to use for the RNG.
+	"""
+
+	def __init__(
+		self,
+		model_name_pair: Tuple[str, pdme.model.DipoleModel],
+		filter: DirectMonteCarloFilter,
+		config: DirectMonteCarloConfig,
+	):
+		self.model_name, self.model = model_name_pair
+
+		# self.measurements = measurements
+		# self.dot_inputs = [(measure.r, measure.f) for measure in self.measurements]
+
+		# self.dot_inputs_array = pdme.measurement.input_types.dot_inputs_to_array(
+		# 	self.dot_inputs
+		# )
+
+		self.config = config
+		self.filter = filter
+		# (
+		# 	self.lows,
+		# 	self.highs,
+		# ) = pdme.measurement.input_types.dot_range_measurements_low_high_arrays(
+		# 	self.measurements
+		# )
+
+	def _single_run(self, seed) -> numpy.ndarray:
+		rng = numpy.random.default_rng(seed)
+
+		sample_dipoles = self.model.get_monte_carlo_dipole_inputs(
+			self.config.monte_carlo_count_per_cycle, -1, rng
+		)
+
+		current_sample = sample_dipoles
+
+		return self.filter.filter_samples(current_sample)
+		# for di, low, high in zip(self.dot_inputs_array, self.lows, self.highs):
+
+		# 	if len(current_sample) < 1:
+		# 		break
+		# 	vals = pdme.util.fast_v_calc.fast_vs_for_dipoleses(
+		# 		numpy.array([di]), current_sample
+		# 	)
+
+		# 	current_sample = current_sample[
+		# 		numpy.all((vals > low) & (vals < high), axis=1)
+		# 	]
+		# return current_sample
+
+	def execute(self) -> DirectMonteCarloResult:
+		step_count = 0
+		total_success = 0
+		total_count = 0
+
+		count_per_step = (
+			self.config.monte_carlo_count_per_cycle * self.config.monte_carlo_cycles
+		)
+		seed_sequence = numpy.random.SeedSequence(self.config.monte_carlo_seed)
+		while (step_count < self.config.max_monte_carlo_cycles_steps) and (
+			total_success < self.config.target_success
+		):
+			_logger.debug(f"Executing step {step_count}")
+			for cycle_i, seed in enumerate(
+				seed_sequence.spawn(self.config.monte_carlo_cycles)
+			):
+				cycle_success_configs = self._single_run(seed)
+				cycle_success_count = len(cycle_success_configs)
+				if cycle_success_count > 0:
+					_logger.debug(
+						f"For cycle {cycle_i} received {cycle_success_count} successes"
+					)
+					_logger.debug(cycle_success_configs)
+					if self.config.write_successes_to_file:
+						sorted_by_freq = numpy.array(
+							[
+								pdme.subspace_simulation.sort_array_of_dipoles_by_frequency(
+									dipole_config
+								)
+								for dipole_config in cycle_success_configs
+							]
+						)
+						dipole_count = numpy.array(cycle_success_configs).shape[1]
+						for n in range(dipole_count):
+							numpy.savetxt(
+								f"{self.config.tag}_{step_count}_{cycle_i}_dipole_{n}.csv",
+								sorted_by_freq[:, n],
+								delimiter=",",
+							)
+				total_success += cycle_success_count
+			_logger.debug(f"At end of step {step_count} have {total_success} successes")
+			step_count += 1
+			total_count += count_per_step
+
+		return DirectMonteCarloResult(
+			successes=total_success,
+			monte_carlo_count=total_count,
+			likelihood=total_success / total_count,
+		)

deepdog/direct_monte_carlo/dmc_filters.py

@@ -0,0 +1,143 @@
+from numpy import ndarray
+from deepdog.direct_monte_carlo.direct_mc import DirectMonteCarloFilter
+from typing import Sequence
+import pdme.measurement
+import pdme.measurement.input_types
+import pdme.util.fast_nonlocal_spectrum
+import pdme.util.fast_v_calc
+import numpy
+
+
+class SingleDotPotentialFilter(DirectMonteCarloFilter):
+	def __init__(self, measurements: Sequence[pdme.measurement.DotRangeMeasurement]):
+		self.measurements = measurements
+		self.dot_inputs = [(measure.r, measure.f) for measure in self.measurements]
+
+		self.dot_inputs_array = pdme.measurement.input_types.dot_inputs_to_array(
+			self.dot_inputs
+		)
+		(
+			self.lows,
+			self.highs,
+		) = pdme.measurement.input_types.dot_range_measurements_low_high_arrays(
+			self.measurements
+		)
+
+	def filter_samples(self, samples: ndarray) -> ndarray:
+		current_sample = samples
+		for di, low, high in zip(self.dot_inputs_array, self.lows, self.highs):
+
+			if len(current_sample) < 1:
+				break
+			vals = pdme.util.fast_v_calc.fast_vs_for_dipoleses(
+				numpy.array([di]), current_sample
+			)
+
+			current_sample = current_sample[
+				numpy.all((vals > low) & (vals < high), axis=1)
+			]
+		return current_sample
+
+
+class DoubleDotSpinQubitFrequencyFilter(DirectMonteCarloFilter):
+	def __init__(
+		self,
+		pair_phase_measurements: Sequence[pdme.measurement.DotPairRangeMeasurement],
+	):
+		self.pair_phase_measurements = pair_phase_measurements
+		self.dot_pair_inputs = [
+			(measure.r1, measure.r2, measure.f)
+			for measure in self.pair_phase_measurements
+		]
+		self.dot_pair_inputs_array = (
+			pdme.measurement.input_types.dot_pair_inputs_to_array(self.dot_pair_inputs)
+		)
+		(
+			self.pair_phase_lows,
+			self.pair_phase_highs,
+		) = pdme.measurement.input_types.dot_range_measurements_low_high_arrays(
+			self.pair_phase_measurements
+		)
+
+	def fast_s_spin_qubit_tarucha_nonlocal_dipoleses(
+		self, dot_pair_inputs: numpy.ndarray, dipoleses: numpy.ndarray
+	) -> numpy.ndarray:
+		"""
+		No error correction here baby.
+		"""
+		ps = dipoleses[:, :, 0:3]
+		ss = dipoleses[:, :, 3:6]
+		ws = dipoleses[:, :, 6]
+
+		r1s = dot_pair_inputs[:, 0, 0:3]
+		r2s = dot_pair_inputs[:, 1, 0:3]
+		f1s = dot_pair_inputs[:, 0, 3]
+		# Don't actually need this
+		# f2s = dot_pair_inputs[:, 1, 3]
+
+		diffses1 = r1s[:, None] - ss[:, None, :]
+		diffses2 = r2s[:, None] - ss[:, None, :]
+
+		norms1 = numpy.linalg.norm(diffses1, axis=3)
+		norms2 = numpy.linalg.norm(diffses2, axis=3)
+
+		alphses1 = (
+			(
+				3
+				* numpy.transpose(
+					numpy.transpose(
+						numpy.einsum("abcd,acd->abc", diffses1, ps) / (norms1**2)
+					)
+					* numpy.transpose(diffses1)
+				)[:, :, :, 0]
+			)
+			- ps[:, :, 0, numpy.newaxis]
+		) / (norms1**3)
+		alphses2 = (
+			(
+				3
+				* numpy.transpose(
+					numpy.transpose(
+						numpy.einsum("abcd,acd->abc", diffses2, ps) / (norms2**2)
+					)
+					* numpy.transpose(diffses2)
+				)[:, :, :, 0]
+			)
+			- ps[:, :, 0, numpy.newaxis]
+		) / (norms2**3)
+
+		bses = (1 / numpy.pi) * (
+			ws[:, None, :] / (f1s[:, None] ** 2 + ws[:, None, :] ** 2)
+		)
+
+		return numpy.einsum("...j->...", alphses1 * alphses2 * bses)
+
+	def filter_samples(self, samples: ndarray) -> ndarray:
+		current_sample = samples
+
+		for pi, plow, phigh in zip(
+			self.dot_pair_inputs_array, self.pair_phase_lows, self.pair_phase_highs
+		):
+			if len(current_sample) < 1:
+				break
+
+			###
+			# This should be  abstracted out, but we're going to dump it here for time pressure's sake
+			###
+			# vals = pdme.util.fast_nonlocal_spectrum.signarg(
+			# 	pdme.util.fast_nonlocal_spectrum.fast_s_nonlocal_dipoleses(
+			# 		numpy.array([pi]), current_sample
+			# 	)
+			#
+			vals = pdme.util.fast_nonlocal_spectrum.signarg(
+				self.fast_s_spin_qubit_tarucha_nonlocal_dipoleses(
+					numpy.array([pi]), current_sample
+				)
+			)
+			current_sample = current_sample[
+				numpy.all(
+					((vals > plow) & (vals < phigh)) | ((vals < plow) & (vals > phigh)),
+					axis=1,
+				)
+			]
+		return current_sample

deepdog/real_spectrum_run.py

@@ -66,6 +66,139 @@ def get_a_result_fast_filter_pairs(input) -> int:
 	return len(current_sample)
 
 
+def get_a_result_fast_filter_potential_pair_phase_only(input) -> int:
+	(
+		model,
+		pair_inputs,
+		pair_phase_lows,
+		pair_phase_highs,
+		monte_carlo_count,
+		seed,
+	) = input
+
+	rng = numpy.random.default_rng(seed)
+	# TODO: A long term refactor is to pull the frequency stuff out from here. The None stands for max_frequency, which is unneeded in the actually useful models.
+	sample_dipoles = model.get_monte_carlo_dipole_inputs(
+		monte_carlo_count, None, rng_to_use=rng
+	)
+
+	current_sample = sample_dipoles
+
+	for pi, plow, phigh in zip(pair_inputs, pair_phase_lows, pair_phase_highs):
+		if len(current_sample) < 1:
+			break
+		vals = pdme.util.fast_nonlocal_spectrum.signarg(
+			pdme.util.fast_nonlocal_spectrum.fast_s_nonlocal_dipoleses(
+				numpy.array([pi]), current_sample
+			)
+		)
+
+		current_sample = current_sample[
+			numpy.all(
+				((vals > plow) & (vals < phigh)) | ((vals < plow) & (vals > phigh)),
+				axis=1,
+			)
+		]
+	return len(current_sample)
+
+
+def get_a_result_fast_filter_tarucha_spin_qubit_pair_phase_only(input) -> int:
+	(
+		model,
+		pair_inputs,
+		pair_phase_lows,
+		pair_phase_highs,
+		monte_carlo_count,
+		seed,
+	) = input
+
+	def fast_s_spin_qubit_tarucha_nonlocal_dipoleses(
+		dot_pair_inputs: numpy.ndarray, dipoleses: numpy.ndarray
+	) -> numpy.ndarray:
+		"""
+		No error correction here baby.
+		"""
+		ps = dipoleses[:, :, 0:3]
+		ss = dipoleses[:, :, 3:6]
+		ws = dipoleses[:, :, 6]
+
+		r1s = dot_pair_inputs[:, 0, 0:3]
+		r2s = dot_pair_inputs[:, 1, 0:3]
+		f1s = dot_pair_inputs[:, 0, 3]
+		# don't actually need, because we're assuming they're the same frequencies across the pair
+		# f2s = dot_pair_inputs[:, 1, 3]
+
+		diffses1 = r1s[:, None] - ss[:, None, :]
+		diffses2 = r2s[:, None] - ss[:, None, :]
+
+		norms1 = numpy.linalg.norm(diffses1, axis=3)
+		norms2 = numpy.linalg.norm(diffses2, axis=3)
+
+		alphses1 = (
+			(
+				3
+				* numpy.transpose(
+					numpy.transpose(
+						numpy.einsum("abcd,acd->abc", diffses1, ps) / (norms1**2)
+					)
+					* numpy.transpose(diffses1)
+				)[:, :, :, 0]
+			)
+			- ps[:, :, 0, numpy.newaxis]
+		) / (norms1**3)
+		alphses2 = (
+			(
+				3
+				* numpy.transpose(
+					numpy.transpose(
+						numpy.einsum("abcd,acd->abc", diffses2, ps) / (norms2**2)
+					)
+					* numpy.transpose(diffses2)
+				)[:, :, :, 0]
+			)
+			- ps[:, :, 0, numpy.newaxis]
+		) / (norms2**3)
+
+		bses = (1 / numpy.pi) * (
+			ws[:, None, :] / (f1s[:, None] ** 2 + ws[:, None, :] ** 2)
+		)
+
+		return numpy.einsum("...j->...", alphses1 * alphses2 * bses)
+
+	rng = numpy.random.default_rng(seed)
+	# TODO: A long term refactor is to pull the frequency stuff out from here. The None stands for max_frequency, which is unneeded in the actually useful models.
+	sample_dipoles = model.get_monte_carlo_dipole_inputs(
+		monte_carlo_count, None, rng_to_use=rng
+	)
+
+	current_sample = sample_dipoles
+
+	for pi, plow, phigh in zip(pair_inputs, pair_phase_lows, pair_phase_highs):
+		if len(current_sample) < 1:
+			break
+
+		###
+		# This should be  abstracted out, but we're going to dump it here for time pressure's sake
+		###
+		# vals = pdme.util.fast_nonlocal_spectrum.signarg(
+		# 	pdme.util.fast_nonlocal_spectrum.fast_s_nonlocal_dipoleses(
+		# 		numpy.array([pi]), current_sample
+		# 	)
+		#
+		vals = pdme.util.fast_nonlocal_spectrum.signarg(
+			fast_s_spin_qubit_tarucha_nonlocal_dipoleses(
+				numpy.array([pi]), current_sample
+			)
+		)
+		current_sample = current_sample[
+			numpy.all(
+				((vals > plow) & (vals < phigh)) | ((vals < plow) & (vals > phigh)),
+				axis=1,
+			)
+		]
+	return len(current_sample)
+
+
 def get_a_result_fast_filter(input) -> int:
 	model, dot_inputs, lows, highs, monte_carlo_count, seed = input
 
@@ -108,6 +241,11 @@ class RealSpectrumRun:
 
 	run_count: int
 	The number of runs to do.
+
+	If pair_measurements is not None, uses pair measurement method (and single measurements too).
+	If pair_phase_measurements is not None, ignores measurements and uses phase measurements _only_
+	This is lazy design on my part.
+
 	"""
 
 	def __init__(
@@ -125,6 +263,9 @@ class RealSpectrumRun:
 		pair_measurements: Optional[
 			Sequence[pdme.measurement.DotPairRangeMeasurement]
 		] = None,
+		pair_phase_measurements: Optional[
+			Sequence[pdme.measurement.DotPairRangeMeasurement]
+		] = None,
 	) -> None:
 		self.measurements = measurements
 		self.dot_inputs = [(measure.r, measure.f) for measure in self.measurements]
@@ -136,6 +277,8 @@ class RealSpectrumRun:
 		if pair_measurements is not None:
 			self.pair_measurements = pair_measurements
 			self.use_pair_measurements = True
+			self.use_pair_phase_measurements = False
+
 			self.dot_pair_inputs = [
 				(measure.r1, measure.r2, measure.f)
 				for measure in self.pair_measurements
@@ -145,8 +288,22 @@ class RealSpectrumRun:
 					self.dot_pair_inputs
 				)
 			)
+		elif pair_phase_measurements is not None:
+			self.use_pair_measurements = False
+			self.use_pair_phase_measurements = True
+			self.pair_phase_measurements = pair_phase_measurements
+			self.dot_pair_inputs = [
+				(measure.r1, measure.r2, measure.f)
+				for measure in self.pair_phase_measurements
+			]
+			self.dot_pair_inputs_array = (
+				pdme.measurement.input_types.dot_pair_inputs_to_array(
+					self.dot_pair_inputs
+				)
+			)
 		else:
 			self.use_pair_measurements = False
+			self.use_pair_phase_measurements = False
 
 		self.models = [model for (_, model) in models_with_names]
 		self.model_names = [name for (name, _) in models_with_names]
@@ -198,6 +355,16 @@ class RealSpectrumRun:
 				self.pair_measurements
 			)
 
+		pair_phase_lows = None
+		pair_phase_highs = None
+		if self.use_pair_phase_measurements:
+			(
+				pair_phase_lows,
+				pair_phase_highs,
+			) = pdme.measurement.input_types.dot_range_measurements_low_high_arrays(
+				self.pair_phase_measurements
+			)
+
 		# define a new seed sequence for each run
 		seed_sequence = numpy.random.SeedSequence(self.initial_seed)
 
@@ -229,6 +396,7 @@ class RealSpectrumRun:
 					seeds = seed_sequence.spawn(self.monte_carlo_cycles)
 
 					if self.use_pair_measurements:
+						_logger.debug("using pair measurements")
 						current_success = sum(
 							pool.imap_unordered(
 								get_a_result_fast_filter_pairs,
@@ -249,6 +417,26 @@ class RealSpectrumRun:
 								self.chunksize,
 							)
 						)
+					elif self.use_pair_phase_measurements:
+						_logger.debug("using pair phase measurements")
+						_logger.debug("specifically using tarucha")
+						current_success = sum(
+							pool.imap_unordered(
+								get_a_result_fast_filter_tarucha_spin_qubit_pair_phase_only,
+								[
+									(
+										model,
+										self.dot_pair_inputs_array,
+										pair_phase_lows,
+										pair_phase_highs,
+										self.monte_carlo_count,
+										seed,
+									)
+									for seed in seeds
+								],
+								self.chunksize,
+							)
+						)
 					else:
 
 						current_success = sum(

deepdog/subset_simulation/subset_simulation_impl.py

@@ -101,11 +101,17 @@ class SubsetSimulation:
 		# _logger.debug(sample_dipoles.shape)
 
 		raw_costs = []
-		_logger.debug(f"Using iterated cost function thing with chunk size {self.initial_cost_chunk_size}")
+		_logger.debug(
+			f"Using iterated cost function thing with chunk size {self.initial_cost_chunk_size}"
+		)
 
 		for x in range(0, len(sample_dipoles), self.initial_cost_chunk_size):
 			_logger.debug(f"doing chunk {x}")
-			raw_costs.extend(self.cost_function_to_use(sample_dipoles[x: x + self.initial_cost_chunk_size]))
+			raw_costs.extend(
+				self.cost_function_to_use(
+					sample_dipoles[x : x + self.initial_cost_chunk_size]
+				)
+			)
 		costs = numpy.array(raw_costs)
 
 		_logger.debug(f"costs: {costs}")
@@ -143,6 +149,37 @@ class SubsetSimulation:
 						delimiter=",",
 					)
 
+				next_seeds_as_array = numpy.array([s for _, s in next_seeds])
+				stdevs = self.get_stdevs_from_arrays(next_seeds_as_array)
+				_logger.info(f"got stdevs: {stdevs.stdevs}")
+				all_long_chains = []
+				for seed_index, (c, s) in enumerate(
+					next_seeds[:: len(next_seeds) // 20]
+				):
+					# chain = mcmc(s, threshold_cost, n_s, model, dot_inputs_array, actual_measurement_array, mcmc_rng, curr_cost=c, stdevs=stdevs)
+					# until new version gotta do
+					_logger.debug(f"\t{seed_index}: doing long chain on the next seed")
+
+					long_chain = self.model.get_mcmc_chain(
+						s,
+						self.cost_function_to_use,
+						1000,
+						threshold_cost,
+						stdevs,
+						initial_cost=c,
+						rng_arg=mcmc_rng,
+					)
+					for _, chained in long_chain:
+						all_long_chains.append(chained)
+				all_long_chains_array = numpy.array(all_long_chains)
+				for n in range(self.model.n):
+					_logger.info(f"{all_long_chains_array[:, n].shape}")
+					numpy.savetxt(
+						f"long_chain_generation_{self.n_c}_{self.n_s}_{i}_dipole_{n}.csv",
+						all_long_chains_array[:, n],
+						delimiter=",",
+					)
+
 			if self.keep_probs_list:
 				for cost_index, cost_chain in enumerate(all_chains[: -self.n_c]):
 					probs_list.append(
@@ -178,7 +215,7 @@ class SubsetSimulation:
 				for cost, chained in chain:
 					try:
 						filtered_cost = cost[0]
-					except IndexError:
+					except (IndexError, TypeError):
 						filtered_cost = cost
 					all_chains.append((filtered_cost, chained))
 			_logger.debug("finished mcmc")

do.sh

@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-# Do - The Simplest Build Tool on Earth.
-# Documentation and examples see https://github.com/8gears/do
-
-set -Eeuo pipefail # -e "Automatic exit from bash shell script on error"  -u "Treat unset variables and parameters as errors"
-
-build() {
-   echo "I am ${FUNCNAME[0]}ing"
-   poetry build
-}
-
-test() {
-   echo "I am ${FUNCNAME[0]}ing"
-   poetry run flake8 deepdog tests
-   poetry run mypy deepdog
-   poetry run pytest
-}
-
-fmt() {
-	poetry run black .
-	find . -not \( -path "./.*" -type d -prune \) -type f -name "*.py" -exec sed -i -e 's/    /\t/g' {} \;
-}
-
-release() {
-   ./scripts/release.sh
-}
-
-htmlcov() {
-	poetry run pytest --cov-report=html
-}
-
-all() {
-   build && test
-}
-
-"$@" # <- execute the task
-
-[ "$#" -gt 0 ] || printf "Usage:\n\t./do.sh %s\n" "($(compgen -A function | grep '^[^_]' | paste -sd '|' -))"

flake.lock

@@ -1,28 +1,33 @@
 {
   "nodes": {
     "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
       "locked": {
-        "lastModified": 1648297722,
-        "narHash": "sha256-W+qlPsiZd8F3XkzXOzAoR+mpFqzm3ekQkJNa+PIh1BQ=",
+        "lastModified": 1710146030,
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "0f8662f1319ad6abf89b3380dd2722369fc51ade",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
         "type": "github"
       },
       "original": {
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "0f8662f1319ad6abf89b3380dd2722369fc51ade",
         "type": "github"
       }
     },
     "flake-utils_2": {
+      "inputs": {
+        "systems": "systems_2"
+      },
       "locked": {
-        "lastModified": 1653893745,
-        "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=",
+        "lastModified": 1705309234,
+        "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1",
+        "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26",
         "type": "github"
       },
       "original": {
@@ -31,29 +36,34 @@
         "type": "github"
       }
     },
+    "nix-github-actions": {
+      "inputs": {
+        "nixpkgs": [
+          "poetry2nixSrc",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1703863825,
+        "narHash": "sha256-rXwqjtwiGKJheXB43ybM8NwWB8rO2dSRrEqes0S7F5Y=",
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "rev": "5163432afc817cf8bd1f031418d1869e4c9d5547",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1655087213,
-        "narHash": "sha256-4R5oQ+OwGAAcXWYrxC4gFMTUSstGxaN8kN7e8hkum/8=",
+        "lastModified": 1710703777,
+        "narHash": "sha256-M4CNAgjrtvrxIWIAc98RTYcVFoAgwUhrYekeiMScj18=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "37b6b161e536fddca54424cf80662bce735bdd1e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "37b6b161e536fddca54424cf80662bce735bdd1e",
-        "type": "github"
-      }
-    },
-    "nixpkgs_2": {
-      "locked": {
-        "lastModified": 1655046959,
-        "narHash": "sha256-gxqHZKq1ReLDe6ZMJSbmSZlLY95DsVq5o6jQihhzvmw=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "07bf3d25ce1da3bee6703657e6a787a4c6cdcea9",
+        "rev": "fc7885fbcea4b782142e06ce2d4d08cf92862004",
         "type": "github"
       },
       "original": {
@@ -62,23 +72,27 @@
         "type": "github"
       }
     },
-    "poetry2nix": {
+    "poetry2nixSrc": {
       "inputs": {
         "flake-utils": "flake-utils_2",
-        "nixpkgs": "nixpkgs_2"
+        "nix-github-actions": "nix-github-actions",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "systems": "systems_3",
+        "treefmt-nix": "treefmt-nix"
       },
       "locked": {
-        "lastModified": 1654921554,
-        "narHash": "sha256-hkfMdQAHSwLWlg0sBVvgrQdIiBP45U1/ktmFpY4g2Mo=",
+        "lastModified": 1708589824,
+        "narHash": "sha256-2GOiFTkvs5MtVF65sC78KNVxQSmsxtk0WmV1wJ9V2ck=",
         "owner": "nix-community",
         "repo": "poetry2nix",
-        "rev": "7b71679fa7df00e1678fc3f1d1d4f5f372341b63",
+        "rev": "3c92540611f42d3fb2d0d084a6c694cd6544b609",
         "type": "github"
       },
       "original": {
         "owner": "nix-community",
         "repo": "poetry2nix",
-        "rev": "7b71679fa7df00e1678fc3f1d1d4f5f372341b63",
         "type": "github"
       }
     },
@@ -86,7 +100,72 @@
       "inputs": {
         "flake-utils": "flake-utils",
         "nixpkgs": "nixpkgs",
-        "poetry2nix": "poetry2nix"
+        "poetry2nixSrc": "poetry2nixSrc"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_3": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "id": "systems",
+        "type": "indirect"
+      }
+    },
+    "treefmt-nix": {
+      "inputs": {
+        "nixpkgs": [
+          "poetry2nixSrc",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1708335038,
+        "narHash": "sha256-ETLZNFBVCabo7lJrpjD6cAbnE11eDOjaQnznmg/6hAE=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "e504621290a1fd896631ddbc5e9c16f4366c9f65",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
       }
     }
   },

flake.nix

@@ -1,63 +1,46 @@
 {
   description = "Application packaged using poetry2nix";
 
-  inputs.flake-utils.url = "github:numtide/flake-utils?rev=0f8662f1319ad6abf89b3380dd2722369fc51ade";
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs?rev=37b6b161e536fddca54424cf80662bce735bdd1e";
-  inputs.poetry2nix.url = "github:nix-community/poetry2nix?rev=7b71679fa7df00e1678fc3f1d1d4f5f372341b63";
+  inputs.flake-utils.url = "github:numtide/flake-utils";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs";
+  inputs.poetry2nixSrc = {
+    url = "github:nix-community/poetry2nix";
+    inputs.nixpkgs.follows = "nixpkgs";
+  };
 
-  outputs = { self, nixpkgs, flake-utils, poetry2nix }:
-    {
-      # Nixpkgs overlay providing the application
-      overlay = nixpkgs.lib.composeManyExtensions [
-        poetry2nix.overlay
-        (final: prev: {
-          # The application
-          deepdog = prev.poetry2nix.mkPoetryApplication {
-            overrides = final.poetry2nix.overrides.withDefaults (self: super: {
-              # …
-              # workaround https://github.com/nix-community/poetry2nix/issues/568
-              pdme = super.pdme.overridePythonAttrs (old: {
-                buildInputs = old.buildInputs or [ ] ++ [ final.python39.pkgs.poetry-core ];
-              });
-            });
-            projectDir = ./.;
-          };
-          deepdogEnv = prev.poetry2nix.mkPoetryEnv {
-            overrides = final.poetry2nix.overrides.withDefaults (self: super: {
-              # …
-              # workaround https://github.com/nix-community/poetry2nix/issues/568
-              pdme = super.pdme.overridePythonAttrs (old: {
-                buildInputs = old.buildInputs or [ ] ++ [ final.python39.pkgs.poetry-core ];
-              });
-            });
-            projectDir = ./.;
-          };
-        })
-      ];
-    } // (flake-utils.lib.eachDefaultSystem (system:
+  outputs = { self, nixpkgs, flake-utils, poetry2nixSrc }:
+    flake-utils.lib.eachDefaultSystem (system:
       let
-        pkgs = import nixpkgs {
-          inherit system;
-          overlays = [ self.overlay ];
+        pkgs = nixpkgs.legacyPackages.${system};
+	poetry2nix = poetry2nixSrc.lib.mkPoetry2Nix { inherit pkgs; };
+      in {
+        packages = {
+	  deepdogApp = poetry2nix.mkPoetryApplication {
+            projectDir = self;
+	    python = pkgs.python39;
+	    preferWheels = true;
 	  };
-      in
-      {
-        apps = {
-          deepdog = pkgs.deepdog;
+	  deepdogEnv = poetry2nix.mkPoetryEnv {
+	    projectDir = self;
+	    python = pkgs.python39;
+	    preferWheels = true;
+	    overrides = poetry2nix.overrides.withDefaults (self: super: {
+	    });
 	  };
-
-        defaultApp = pkgs.deepdog;
-        devShell = pkgs.mkShell {
+	  default = self.packages.${system}.deepdogEnv;
+	};
+	devShells.default = pkgs.mkShell {
+	  inputsFrom = [ self.packages.${system}.deepdogEnv ];
 	  buildInputs = [
 	    pkgs.poetry
-            pkgs.deepdogEnv
-            pkgs.deepdog
+	    self.packages.${system}.deepdogEnv
+	    self.packages.${system}.deepdogApp
+	    pkgs.just
 	  ];
 	  shellHook = ''
 	    export DO_NIX_CUSTOM=1
 	  '';
-          packages = [ pkgs.nodejs-16_x ];
 	};
-
-      }));
+      }
+    );
 }

justfile

@@ -0,0 +1,54 @@
+
+# execute default build
+default: build
+
+# builds the python module using poetry
+build:
+	echo "building..."
+	poetry build
+
+# print a message displaying whether nix is being used
+checknix:
+	#!/usr/bin/env bash
+	set -euxo pipefail
+	if [[ "${DO_NIX_CUSTOM:=0}" -eq 1 ]]; then
+		echo "In an interactive nix env."
+	else
+		echo "Using poetry as runner, no nix detected."
+	fi
+
+# run all tests
+test: fmt
+	#!/usr/bin/env bash
+	set -euxo pipefail
+	
+	if [[ "${DO_NIX_CUSTOM:=0}" -eq 1 ]]; then
+		echo "testing, using nix..."
+		flake8 deepdog tests
+		mypy deepdog
+		pytest
+	else
+		echo "testing..."
+		poetry run flake8 deepdog tests
+		poetry run mypy deepdog
+		poetry run pytest
+	fi
+
+# format code
+fmt:
+	#!/usr/bin/env bash
+	set -euxo pipefail
+	if [[ "${DO_NIX_CUSTOM:=0}" -eq 1 ]]; then
+	      black .
+	else
+		poetry run black .
+	fi
+	find deepdog -type f -name "*.py" -exec sed -i -e 's/    /\t/g' {} \;
+	find tests -type f -name "*.py" -exec sed -i -e 's/    /\t/g' {} \;
+
+# release the app, checking that our working tree is clean and ready for release
+release:
+	./scripts/release.sh
+
+htmlcov:
+	poetry run pytest --cov-report=html

pyproject.toml

@@ -1,12 +1,12 @@
 [tool.poetry]
 name = "deepdog"
-version = "0.7.4"
+version = "0.7.9"
 description = ""
 authors = ["Deepak Mallubhotla <dmallubhotla+github@gmail.com>"]
 
 [tool.poetry.dependencies]
 python = ">=3.8.1,<3.10"
-pdme = "^0.9.1"
+pdme = "^0.9.3"
 numpy = "1.22.3"
 scipy = "1.10"
 

tests/test_bayes_run_with_ss.py

@@ -151,7 +151,7 @@ def test_bayesss_with_tighter_cost(snapshot):
 		ss_default_upper_w_log_step=4,
 		ss_dump_last_generation=False,
 		write_output_to_bayesruncsv=False,
-		ss_initial_costs_chunk_size=1
+		ss_initial_costs_chunk_size=1,
 	)
 	result = square_run.go()