# This file is part of Pynguin.
#
# SPDX-FileCopyrightText: 2019–2026 Pynguin Contributors
#
# SPDX-License-Identifier: MIT
#
"""Provides a configuration interface for the test generator."""
import dataclasses
import enum
import time
from pynguin.utils.statistics.runtimevariable import RuntimeVariable
[docs]
class ExportStrategy(str, enum.Enum):
"""Contains all available export strategies.
These strategies allow to export the generated test cases in different styles,
such as the style of the `PyTest` framework. Setting the value to `NONE` will
prevent exporting of the generated test cases (only reasonable for
benchmarking, though).
"""
PY_TEST = "PY_TEST"
"""Export tests in the style of the PyTest framework."""
NONE = "NONE"
"""Do not export test cases at all."""
[docs]
class Algorithm(str, enum.Enum):
"""Different algorithms supported by Pynguin."""
DYNAMOSA = "DYNAMOSA"
"""The dynamic many-objective sorting algorithm (cf. Panichella et al. Automated
test case generation as a many-objective optimisation problem with dynamic selection
of the targets. TSE vol. 44 issue 2)."""
LLM = "LLM"
"""Query a large language model for tests. NOTE: This does not execute the usual test and
assertion generation stages of Pynguin but queries an LLM and writes the resulting test cases to
the output file."""
MIO = "MIO"
"""The MIO test suite generation algorithm (cf. Andrea Arcuri. Many Independent
Objective (MIO) Algorithm for Test Suite Generation. Proc. SBSE 2017)."""
MOSA = "MOSA"
"""The many-objective sorting algorithm (cf. Panichella et al. Reformulating Branch
Coverage as a Many-Objective Optimization Problem. Proc. ICST 2015)."""
LLMOSA = "LLMOSA"
"""The many-objective sorting algorithm with LLM."""
LLDYNAMOSA = "LLDYNAMOSA"
"""The dynamic many-objective sorting algorithm with LLM."""
RANDOM = "RANDOM"
"""A feedback-direct random test generation approach similar to the algorithm
proposed by Randoop (cf. Pacheco et al. Feedback-directed random test generation.
Proc. ICSE 2007)."""
RANDOM_TEST_SUITE_SEARCH = "RANDOM_TEST_SUITE_SEARCH"
"""Performs random search on test suites."""
RANDOM_TEST_CASE_SEARCH = "RANDOM_TEST_CASE_SEARCH"
"""Performs random search on test cases."""
WHOLE_SUITE = "WHOLE_SUITE"
"""A whole-suite test generation approach similar to the one proposed by EvoSuite
(cf. Fraser and Arcuri. EvoSuite: Automatic Test Suite Generation for
Object-Oriented Software. Proc. ESEC/FSE 2011).
This algorithm can be modified to use an archive (cf. Rojas, José Miguel, et al.
"A detailed investigation of the effectiveness of whole test suite generation."
Empirical Software Engineering 22.2 (2017): 852-893.), by using the
following options: --use-archive True, --seed-from-archive True and
--filter-covered-targets-from-test-cluster True.
"""
[docs]
class AssertionGenerator(str, enum.Enum):
"""Different approaches for assertion generation supported by Pynguin."""
LLM = "LLM"
"""Use Large Language Model(LLM) for assertion generation."""
MUTATION_ANALYSIS = "MUTATION_ANALYSIS"
"""Use the mutation analysis approach for assertion generation."""
CHECKED_MINIMIZING = "CHECKED_MINIMIZING"
"""All assertions that do not increase the checked coverage are removed."""
SIMPLE = "SIMPLE"
"""Use the simple approach for primitive and none assertion generation."""
NONE = "NONE"
"""Do not create any assertions."""
[docs]
class MutationStrategy(str, enum.Enum):
"""Different strategies for creating mutants.
Only respected when using the MUTATION_ANALYSIS approach for assertion generation.
"""
FIRST_ORDER_MUTANTS = "FIRST_ORDER_MUTANTS"
"""Generate first order mutants."""
FIRST_TO_LAST = "FIRST_TO_LAST"
"""Higher order mutation strategy FirstToLast.
(cf. Mateo et al. Validating Second-Order Mutation at System Level. Article.
IEEE Transactions on SE 39.4 2013)"""
BETWEEN_OPERATORS = "BETWEEN_OPERATORS"
"""Higher order mutation strategy BetweenOperators.
(cf. Mateo et al. Validating Second-Order Mutation at System Level. Article.
IEEE Transactions on SE 39.4 2013)"""
RANDOM = "RANDOM"
"""Higher order mutation strategy Random.
(cf. Mateo et al. Validating Second-Order Mutation at System Level. Article.
IEEE Transactions on SE 39.4 2013)"""
EACH_CHOICE = "EACH_CHOICE"
"""Higher order mutation strategy EachChoice.
(cf. Mateo et al. Validating Second-Order Mutation at System Level. Article.
IEEE Transactions on SE 39.4 2013)"""
[docs]
class RefinementGranularity(str, enum.Enum):
"""Granularity of the LLM readability & assertion refinement stages."""
COMBINED = "combined"
"""One module-level prompt that does both readability and assertion refinement
(default; most token-efficient, highest output variance)."""
MODULE_SEPARATE = "module_separate"
"""Two module-level prompts: readability first, then semantic assertions."""
PER_TEST = "per_test"
"""Per-test readability and assertion prompts (previous behavior, most robust)."""
[docs]
class MutationStrengtheningGranularity(str, enum.Enum):
"""Granularity of mutation-driven assertion strengthening."""
FULL_MODULE = "full_module"
"""Module-level prompt batching surviving mutants into a full-module request (default)."""
PER_TEST = "per_test"
"""Per-test mutation strengthening loop (legacy behavior)."""
[docs]
class LLMMode(str, enum.Enum):
"""The execution mode for LLM queries in search algorithms."""
SYNC = "sync"
"""Synchronous execution: algorithms block waiting for LLM query responses."""
ASYNC = "async"
"""Asynchronous execution: non-blocking queries dispatched in the background."""
[docs]
class MinimizationDirection(str, enum.Enum):
"""Directions for test case minimization.
Either start with the first statement and try removing it and all forward dependent
statements, or start with the last statement and try removing it and all backward
dependent statements.
"""
FORWARD = "FORWARD"
"""Apply forward minimization, which removes statements from the beginning to the end."""
BACKWARD = "BACKWARD"
"""Apply backward minimization, which removes statements from the end to the beginning."""
[docs]
class MinimizationStrategy(str, enum.Enum):
"""Different strategies for minimizing test cases."""
NONE = "NONE"
"""Do not apply any minimization."""
CASE = "CASE"
"""Apply minimization at the test case level."""
SUITE = "SUITE"
"""Apply minimization at the test suite level, i.e., remove entire test cases that
do not contribute to the coverage of the module under test."""
COMBINED = "COMBINED"
"""Apply minimization at the test case level but check against all test cases in the suite."""
[docs]
class TypeInferenceStrategy(str, enum.Enum):
"""The different available type-inference strategies."""
NONE = "NONE"
"""Ignore any type information given in the module under test."""
TYPE_HINTS = "TYPE_HINTS"
"""Use type information from type hints in the module under test."""
LLM = "LLM"
"""Use an LLM to infer types for the module under test."""
LLM_WITH_SUBTYPES = "LLM_WITH_SUBTYPES"
"""Use an LLM to infer types and subtypes for the module under test."""
TYPEEVALPY = "TYPEEVALPY"
"""Use TypeEvalPy data only for type inference."""
TYPESHED = "TYPESHED"
"""Use type hints where available, falling back to typeshed stub information
for callables without runtime annotations (e.g. C-accelerated stdlib code)."""
[docs]
class SubtypeInferenceStrategy(str, enum.Enum):
"""The different available type-inference strategies."""
NONE = "NONE"
"""Do not infer subtypes."""
STRING = "STRING"
"""Infer subtypes for strings."""
[docs]
class ElementVisibility(str, enum.Enum):
"""Which elements of a module are considered for test generation."""
PUBLIC = "PUBLIC"
"""Only include public elements (default, current behavior)."""
PROTECTED = "PROTECTED"
"""Include public and protected elements (single leading underscore)."""
ALL = "ALL"
"""Include public, protected, and private elements (double leading
underscore, including name-mangled)."""
[docs]
class StatisticsBackend(str, enum.Enum):
"""The different available statistics backends to write statistics."""
NONE = "NONE"
"""Do not write any statistics."""
CONSOLE = "CONSOLE"
"""Write statistics to the standard out."""
CSV = "CSV"
"""Write statistics to a CSV file."""
[docs]
class CoverageMetric(str, enum.Enum):
"""The different available coverage metrics available for optimisation."""
BRANCH = "BRANCH"
"""Calculate how many of the possible branches in the code were executed"""
LINE = "LINE"
"""Calculate how many of the possible lines in the code were executed"""
CHECKED = "CHECKED"
"""Calculate how many of the possible lines in the
code are checked by an assertion."""
[docs]
class Selection(str, enum.Enum):
"""Different selection algorithms to select from."""
RANDOM_SELECTION = "RANDOM_SELECTION"
"""Random selection."""
RANK_SELECTION = "RANK_SELECTION"
"""Rank selection."""
TOURNAMENT_SELECTION = "TOURNAMENT_SELECTION"
"""Tournament selection. Use `tournament_size` to set size."""
[docs]
@dataclasses.dataclass
class StatisticsOutputConfiguration:
"""Configuration related to output."""
report_dir: str = "pynguin-report"
"""Directory in which to put HTML and CSV reports"""
statistics_backend: StatisticsBackend = StatisticsBackend.CSV
"""Which backend to use to collect data"""
timeline_interval: int = 1 * 1_000_000_000
"""Time interval in nano-seconds for timeline statistics, i.e., we select a data
point after each interval. This can be interpolated, if there is no exact
value stored at the time-step of the interval, see `timeline_interpolation`.
The default value is every 1.00s."""
timeline_interpolation: bool = True
"""Interpolate timeline values"""
output_variables: list[RuntimeVariable] = dataclasses.field(
default_factory=lambda: [
RuntimeVariable.TargetModule,
RuntimeVariable.Coverage,
]
)
"""List of variables to output to the statistics backend."""
configuration_id: str = ""
"""Label that identifies the used configuration of Pynguin. This is only done
when running experiments."""
run_id: str = ""
"""Id of the cluster run. Useful for finding the log entries that belong to a
certain result."""
project_name: str = ""
"""Label that identifies the project name of Pynguin. This is useful when
running experiments."""
create_coverage_report: bool = False
"""Create a coverage report for the tested module.
This can be helpful to find hard to cover parts because Pynguin measures coverage
on bytecode level which might yield different results when compared with other
tools, e.g., Coverage.py."""
type_guess_top_n: int = 10
"""When exporting type guesses for parameters, how many guesses per parameter
should be exported? Expects positive integers."""
store_test_before_execution: bool = False
"""When enabled, each test case is written to a file before execution and removed
after successful completion. If a test causes a crash, the file remains.
Use with caution: Converting pynguin internal representation to a pytest test case
and writing it to a file is expensive and leads to lower iterations/coverage."""
[docs]
@dataclasses.dataclass
class Minimization:
"""Configuration for test case minimization."""
test_case_minimization_strategy: MinimizationStrategy = MinimizationStrategy.CASE
"""Strategy to apply for minimizing test cases to remove redundant statements
while preserving fitness."""
test_case_minimization_direction: MinimizationDirection = MinimizationDirection.BACKWARD
"""Direction to apply for minimizing test cases."""
[docs]
@dataclasses.dataclass
class TestCaseOutputConfiguration:
"""Configuration related to test case output."""
output_path: str
"""Path to an output folder for the generated test cases."""
crash_path: str = ""
"""Path to an output folder for the generated test cases that caused a crash.
Only works when running in a subprocess. If set to "", the are stored in the
output_path."""
export_strategy: ExportStrategy = ExportStrategy.PY_TEST
"""The export strategy determines for which test-runner system the
generated tests should fit."""
max_length_test_case: int = 2500
"""The maximum number of statement in as test case (normal + assertion
statements)"""
assertion_generation: AssertionGenerator = AssertionGenerator.MUTATION_ANALYSIS
"""The generator that shall be used for assertion generation."""
allow_stale_assertions: bool = False
"""Allow assertion on things that did not change between statement executions."""
filter_assertions_in_subprocess: bool = True
"""Run the assertion-filtering executions in a fresh subprocess. Filtering in a
separate process exercises per-process nondeterminism (object ``id()``, identity
hashing, ``hash()`` of str/bytes, once-computed timestamps) that an in-process
re-execution cannot observe, so assertions hard-coding such values are dropped
instead of surviving to fail in a downstream ``pytest`` run."""
mutation_strategy: MutationStrategy = MutationStrategy.FIRST_ORDER_MUTANTS
"""The strategy that shall be used for creating mutants in the mutation analysis
assertion generation method."""
mutation_order: int = 1
"""The order of the generated higher order mutants in the mutation analysis
assertion generation method."""
maximum_llm_assertion_time: int = -1
"""Maximum wall-clock time in seconds for generating assertions via LLM
(-1 = unlimited). When the budget is exceeded, no further test cases are
queried."""
maximum_mutation_time: int = -1
"""Maximum wall-clock time in seconds for executing tests against mutants
during assertion generation (-1 = unlimited). When the budget is exceeded the
current mutant finishes, then no further mutants are checked."""
maximum_mutants: int = -1
"""Maximum number of mutants checked during assertion generation. If the
module yields more mutants, a seeded random sample of this size is used
(-1 = unlimited)."""
post_process: bool = True
"""Should the results be post processed? For example, truncate test cases after
statements that raise an exception."""
minimization: Minimization = dataclasses.field(
default_factory=lambda: Minimization(
test_case_minimization_strategy=MinimizationStrategy.CASE,
test_case_minimization_direction=MinimizationDirection.BACKWARD,
)
)
"""Strategy to apply for minimizing test cases."""
float_precision: float = 0.01
"""Precision to use in float comparisons and assertions"""
format_with_black: bool = True
"""Format the generated test cases using black."""
no_xfail: bool = False
"""If True, unexpected exceptions will be wrapped with pytest.raises() instead of
marking the test with @pytest.mark.xfail(strict=True). Use this for better mutation
scores."""
assertion_minimization: bool = True
"""If enabled, remove redundant assertions after mutation analysis, keeping a
minimal subset that preserves the set of killed mutants (greedy set cover)."""
[docs]
@dataclasses.dataclass
class SeedingConfiguration:
"""Configuration related to seeding."""
seed: int = time.time_ns()
"""A predefined seed value for the random number generator that is used."""
constant_seeding: bool = True
"""Should the generator use a static constant seeding technique to improve constant
generation?"""
initial_population_seeding: bool = False
"""Should the generator use previously existing testcases to seed the initial
population?"""
initial_population_data: str = ""
"""The path to the file with the pre-existing tests. The path has to include the
file itself."""
seeded_testcases_reuse_probability: float = 0.9
"""Probability of using seeded testcases when initial population seeding is
enabled."""
initial_population_mutations: int = 0
"""Number of how often the testcases collected by initial population seeding should
be mutated to promote diversity"""
dynamic_constant_seeding: bool = True
"""Enables seeding of constants at runtime."""
seeded_primitives_reuse_probability: float = 0.2
"""Probability for using seeded primitive values instead of randomly
generated ones."""
seeded_dynamic_values_reuse_probability: float = 0.6
"""Probability of using dynamically seeded values when a primitive seeded
value will be used."""
seed_from_archive: bool = False
"""When sampling new test cases reuse some from the archive, if one is used."""
seed_from_archive_probability: float = 0.2
"""Instead of creating a new test case, reuse a covering solution from the archive,
iff an archive is used."""
seed_from_archive_mutations: int = 3
"""Number of mutations applied when sampling from the archive."""
max_dynamic_length: int = 1000
"""Maximum length of strings/bytes that should be stored in the dynamic constant
pool."""
max_dynamic_pool_size: int = 50
"""Maximum number of constants of the same type that should be stored in the
dynamic constant pool."""
[docs]
@dataclasses.dataclass
class MIOPhaseConfiguration:
"""Configuration for a phase of MIO."""
number_of_tests_per_target: int
"""Number of test cases for each target goal to keep in an archive."""
random_test_or_from_archive_probability: float
"""Probability [0,1] of sampling a new test at random or choose an existing one in
an archive."""
number_of_mutations: int
"""Number of mutations allowed to be done on the same individual before
sampling a new one."""
[docs]
@dataclasses.dataclass
class MIOConfiguration:
"""Configuration that is specific to the MIO approach."""
initial_config: MIOPhaseConfiguration = dataclasses.field(
default_factory=lambda: MIOPhaseConfiguration(
number_of_tests_per_target=10,
random_test_or_from_archive_probability=0.5,
number_of_mutations=1,
)
)
"""Configuration to use before focused phase."""
focused_config: MIOPhaseConfiguration = dataclasses.field(
default_factory=lambda: MIOPhaseConfiguration(
number_of_tests_per_target=1,
random_test_or_from_archive_probability=0.0,
number_of_mutations=10,
)
)
"""Configuration to use in focused phase"""
exploitation_starts_at_percent: float = 0.5
"""Percentage ]0,1] of search budget after which exploitation is activated, i.e.,
switching to focused phase."""
[docs]
@dataclasses.dataclass
class RandomConfiguration:
"""Configuration that is specific to the RANDOM approach."""
max_sequence_length: int = 10
"""The maximum length of sequences that are generated, 0 means infinite."""
max_sequences_combined: int = 10
"""The maximum number of combined sequences, 0 means infinite."""
[docs]
class MigrationStrategy(str, enum.Enum):
"""Which island-migration mechanism(s) are active."""
DISABLED = "DISABLED"
"""No migration; islands search in complete isolation. Ablation baseline."""
GOAL_TRIGGERED = "GOAL_TRIGGERED"
"""Broadcast newly-covered-parent test cases between islands, event-triggered."""
PERIODIC = "PERIODIC"
"""Every F local generations, send K migrants to the ring neighbor only."""
COMBINED = "COMBINED"
"""Both mechanisms run independently: a test case can be sent immediately via
goal-triggered broadcast and, separately, be selected as a periodic migrant
later. The shared content-hash dedup set prevents redundant local execution
when the same content arrives via both paths."""
[docs]
class MigrantSelectionPolicy(str, enum.Enum):
"""How periodic migration selects the individuals sent to the ring neighbor."""
RANDOM = "RANDOM"
"""Selects K individuals uniformly at random, with replacement."""
BEST = "BEST"
"""Selects the K individuals with the highest fitness value from the
recently evolved population."""
RANK = "RANK"
"""Whitley's linear rank selection: draws K individuals (with replacement)
from the population ordered best-to-worst, with selection probability
decreasing linearly by rank."""
[docs]
class PopulationAllocation(str, enum.Enum):
"""How the reference sequential population size is allocated across islands."""
FULL_PER_ISLAND = "FULL_PER_ISLAND"
"""Every island gets the full reference population size."""
FIXED_TOTAL_DISTRIBUTED = "FIXED_TOTAL_DISTRIBUTED"
"""The reference population size is the aggregate across all islands,
distributed as evenly as possible."""
[docs]
@dataclasses.dataclass
class IslandConfiguration:
"""Configuration for island-model parallel search.
This configuration is independent of the selected search algorithm, and each
island runs a normal instance of it; setting num_islands > 1 spawns that many
concurrent island processes instead of a single one.
"""
num_islands: int = 1
"""Number of island processes to run concurrently. 1 disables island mode."""
migration_strategy: MigrationStrategy = MigrationStrategy.DISABLED
"""Which migration mechanism(s) are active. Requires algorithm to be DYNAMOSA
or LLDYNAMOSA. DISABLED isolates islands completely, for comparison against
migration's effect."""
periodic_migration_frequency: int = 5
"""F: send periodic migrants every F local generations. Only used when
migration_strategy is PERIODIC or COMBINED."""
periodic_migration_size: int = 5
"""K: number of individuals sent to the ring neighbor per periodic migration
event. Only used when migration_strategy is PERIODIC or COMBINED."""
migrant_selection_policy: MigrantSelectionPolicy = MigrantSelectionPolicy.RANDOM
"""How periodic migrants are selected. Only used when migration_strategy is
PERIODIC or COMBINED."""
migrant_rank_bias: float = 2.0
"""Bias for better individuals in rank selection of periodic migrants, in the
range (1, 2]. The default 2.0 is the strongest selection pressure. Only used
when migrant_selection_policy is RANK."""
population_allocation: PopulationAllocation = PopulationAllocation.FULL_PER_ISLAND
"""How the reference sequential population size is allocated across islands."""
island_id: int = -1
"""Set per-island by the orchestrator before spawning; -1 in the base config."""
[docs]
class ImmigrationRouting(str, enum.Enum):
"""Where a completed LLM-worker query result gets delivered."""
TARGETED = "TARGETED"
"""Deliver only to the island named in the queried (island, callable) pair."""
BROADCAST = "BROADCAST"
"""Deliver to every island via the same migration mechanism peer-covered test
cases use. Requires island.migration_strategy to not be DISABLED."""
[docs]
@dataclasses.dataclass
class LLMWorkerConfiguration:
"""Configuration for the centralized asynchronous LLM query worker.
A dedicated process that islands report under-covered callables to; it queries
the LLM asynchronously and routes results back per immigration_routing.
"""
enabled: bool = False
"""Run a dedicated async LLM query worker alongside the islands. Requires
algorithm=LLDYNAMOSA and island.num_islands > 1."""
max_in_flight_requests: int = 1
"""Maximum number of LLM requests the worker keeps in flight at once.
1 means a request completes before the next is sent."""
immigration_routing: ImmigrationRouting = ImmigrationRouting.TARGETED
"""Where completed query results are delivered."""
cost_per_1k_input_tokens: float = 0.0
"""USD cost per 1,000 input tokens, for estimating query cost. 0 disables cost
estimation (only counts/latency are tracked)."""
cost_per_1k_output_tokens: float = 0.0
"""USD cost per 1,000 output tokens, for estimating query cost."""
[docs]
@dataclasses.dataclass
class TypeInferenceConfiguration:
"""Configuration related to type inference."""
type_inference_strategy: TypeInferenceStrategy = TypeInferenceStrategy.TYPE_HINTS
"""The strategy for type-inference that shall be used"""
type_tracing: bool | float = 0.0
"""Probability to trace usage of parameters with unknown types to improve type
guesses during test execution. Type tracing requires a separate second.
The value should be a float in [0,1]. Boolean is kept for backwards compatibility
as Python internally converts True to 1.0 and False to 0.0 anyways."""
subtype_inference: SubtypeInferenceStrategy = SubtypeInferenceStrategy.NONE
"""The strategy for subtype-inference that shall be used."""
type_tracing_subtype_weight: float = 0.3
"""Weight for selecting the subtype inference strategy for type selection during type
tracing."""
type_tracing_argument_type_weight: float = 0.5
"""Weight for selecting the argument type inference strategy for type selection
during type tracing."""
type_tracing_attribute_weight: float = 0.2
"""Weight for selecting the attribute table inference strategy for type selection
during type tracing."""
typeevalpy_json_path: str = ""
"""Path to a TypeEvalPy JSON file containing type information to use during test
generation. If provided, the file will be parsed and the type information will be
used to improve type inference."""
[docs]
@dataclasses.dataclass
class PynguinMLConfiguration:
"""Configurations for PynguinML, specifically for machine learning API testing."""
ml_testing_enabled: bool = False
"""
Enables ML-based test generation using API constraints. When set to True, PynguinML
will use constraints and other ML-specific behaviours during input generation.
Note: 'constraints_path' must also be set to locate the constraints files.
"""
constraints_path: str = ""
"""
Directory path to YAML/JSON constraints files.
"""
dtype_mapping_path: str = ""
"""
Path to a YAML/JSON file that maps library-specific datatypes to NumPy datatypes.
Used by PynguinML for input generation. Specify only valid NumPy dtypes without
the "np." prefix. Example: "torch.int32" -> "int32".
"""
constructor_function: str = ""
"""
Optional constructor function to build tensors (e.g., torch.tensor or
tensorflow.convert_to_tensor). Must accept a np.ndarray as input.
Note: Also set 'constructor_function_parameter' to specify which parameter
receives the np.ndarray.
"""
constructor_function_parameter: str = ""
"""
Name of the parameter in the constructor function that receives the np.ndarray.
Must be set when 'constructor_function' is specified.
"""
max_ndim: int = 4
"""
Maximum number of dimensions of the tensors.
If set too big, it can cause memory errors due to large tensor sizes.
"""
max_shape_dim: int = 4
"""
Maximum size of a dimension of a shape.
If set too big, it can cause memory errors due to large tensor sizes.
"""
ignore_constraints_probability: float = 0.25
"""
Probability of ignoring constraints and falling back to Pynguin's default
behavior when constructing statements.
"""
[docs]
@dataclasses.dataclass
class TestCreationConfiguration:
"""Configuration related to test creation."""
max_recursion: int = 10
"""Recursion depth when trying to create objects in a test case."""
max_delta: int = 20
"""Maximum size of delta for numbers during mutation"""
max_int: int = 2048
"""Maximum size of randomly generated integers (minimum range = -1 * max)"""
string_length: int = 20
"""Maximum length of randomly generated strings"""
bytes_length: int = 20
"""Maximum length of randomly generated bytes"""
collection_size: int = 5
"""Maximum length of randomly generated collections"""
collection_reference_probability: float = 0.5
"""Probability that a collection element slot uses a reference to an existing
in-scope variable instead of a fresh literal, when a reference pool is
available. Expects values in [0,1]."""
track_collection_accesses: bool = True
"""Whether to track collection accesses during test execution and bound
mutated collections (lists, tuples, dicts) based on observed accesses."""
primitive_reuse_probability: float = 0.5
"""Probability to reuse an existing primitive in a test case, if available.
Expects values in [0,1]"""
object_reuse_probability: float = 0.9
"""Probability to reuse an existing object in a test case, if available.
Expects values in [0,1]"""
none_weight: float = 1
"""Weight to use None as parameter type during test generation.
Expects values > 0."""
any_weight: float = 5
"""Weight to use Any as parameter type during test generation.
Expects values > 0."""
original_type_weight: float = 5
"""Weight to use the originally annotated type as parameter type during test
generation. Expects values > 0."""
type_tracing_weight: float = 10
"""Weight to use the type guessed from type tracing as parameter type during
test generation. Expects values > 0."""
type_tracing_kept_guesses: int = 2
"""Amount of kept recently guessed types per parameter, when type tracing
is used."""
wrap_var_param_type_probability: float = 0.7
"""Probability to wrap the type required for a ``*arg`` or ``**kwargs`` parameter
in a list or dict, respectively. Expects values in [0,1]"""
negate_type: float = 0.1
"""When inferring a type from proxies, it may also be desirable to negate the chosen
type, e.g., such that an instance check or a getattr() evaluate to False.
Expects values in [0,1]"""
skip_optional_parameter_probability: float = 0.7
"""Probability to skip an optional parameter, i.e., do not fill such a parameter."""
callable_argument_probability: float = 0.25
"""Probability to satisfy a parameter of unknown (``Any``) type with a callable
value -- a cluster function, a class, a builtin such as ``str``, or a synthesized
``lambda`` -- instead of a literal. Parameters that are explicitly annotated as
``Callable`` always receive a callable value, independent of this setting.
Expects values in [0,1]."""
callable_invocation_probability: float = 0.25
"""Probability to follow a statement whose value may be a callable (an unannotated
or ``Callable``-typed return value) with a statement that invokes it. This is what
reaches the body of a closure returned by a higher-order function.
Expects values in [0,1]."""
max_attempts: int = 1000
"""Number of attempts when generating an object before giving up"""
insertion_uut: float = 0.5
"""Score for selection of insertion of UUT calls"""
max_size: int = 100
"""Maximum number of test cases in a test suite"""
use_random_object_for_call: float = 0.1
"""When adding or modifying a call on an object, use a random modifier instead
of only modifiers for that type. Expects values in [0, 1]."""
generate_field_statements: bool = False
"""Whether to discover public fields/properties of analysed classes and emit
attribute-access statements (``var_1 = var_0.field``) for them. Off by default
because it is new behaviour relative to the class-based representation, where
fields only ever entered generation through the LLM subsystem."""
[docs]
@dataclasses.dataclass
class GeneratorSelectionConfiguration:
"""Configuration related to selecting a generator for a certain type."""
generator_selection_algorithm: Selection = Selection.RANK_SELECTION
"""The selection algorithm to use for selecting the generator for a statement.
Currently only RANK_SELECTION is supported."""
generator_selection_bias: float = 1.7
"""The bias to use for rank selection when selecting the generator for a statement.
Expects values in [1.0, 2.0]. 1.0 is random selection, higher values bias towards
higher ranked generators. A value above 2.0 leads to cutting off the lower ranked
generators."""
generator_any_distance: int = 30
"""The distance to and from the any type to any other type in the type hierarchy.
As the any type is a super- and subtype of all type but it is most likely not
the type we want, we want to punish it in the type selection process."""
generator_not_constructor_penalty: float = 10.0
"""Penalty for selecting a generator that is not a __init__ method."""
generator_param_penalty: float = 1.0
"""Penalty for each parameter of the generator."""
generator_hierarchy_penalty: float = 1.0
"""Penalty for the distance between the generators type and the desired type.
The penalty is multiplied with the distance in the type hierarchy."""
generator_any_type_penalty: float = 100.0
"""Penalty for selecting a generator that returns only Any.
While Any is always a valid type, we still want to avoid it as a generator."""
[docs]
@dataclasses.dataclass
class SearchAlgorithmConfiguration:
"""General configuration for search algorithms."""
min_initial_tests: int = 1
"""Minimum number of tests in initial test suites"""
max_initial_tests: int = 10
"""Maximum number of tests in initial test suites"""
population: int = 10
"""Population size of genetic algorithm"""
chromosome_length: int = 48
"""Maximum length of chromosomes during search"""
coverage_metrics: list[CoverageMetric] = dataclasses.field(
default_factory=lambda: [CoverageMetric.BRANCH]
)
"""List of coverage metrics that are optimised during the search"""
chop_max_length: bool = True
"""Chop statements after exception if length has reached maximum"""
elite: int = 1
"""Elite size for search algorithm"""
crossover_rate: float = 0.648
"""Probability of crossover"""
test_insertion_probability: float = 0.1
"""Initial probability of inserting a new test in a test suite"""
test_delete_probability: float = 1.0 / 3.0
"""Probability of deleting statements during mutation"""
test_change_probability: float = 1.0 / 3.0
"""Probability of changing statements during mutation"""
test_insert_probability: float = 1.0 / 3.0
"""Probability of inserting new statements during mutation"""
statement_insertion_probability: float = 0.5
"""Initial probability of inserting a new statement in a test case"""
random_perturbation: float = 0.2
"""Probability to replace a primitive with a random new value rather than adding
a delta."""
change_parameter_probability: float = 0.1
"""Probability of replacing parameters when mutating a method or constructor
statement in a test case. Expects values in [0,1]"""
change_statement_type_probability: float = 0.05
"""Probability, per mutated statement, of replacing it with a statement of a
different generated type (keeping its bound variable) instead of the regular
value/call mutation. Expects values in [0,1]."""
tournament_size: int = 4
"""Number of individuals for tournament selection."""
rank_bias: float = 1.68
"""Bias for better individuals in rank selection"""
selection: Selection = Selection.RANK_SELECTION
"""The selection operator for genetic algorithms."""
use_archive: bool = False
"""Some algorithms can be enhanced with an optional archive, e.g. Whole Suite ->
Whole Suite + Archive. Use this option to enable the usage of an archive.
Algorithms that always use an archive are not affected by this option."""
filter_covered_targets_from_test_cluster: bool = False
"""Focus search by filtering out elements from the test cluster when
they are fully covered."""
number_of_mutations: int = 3
"""Number of mutations that should be applied in one breeding step."""
[docs]
@dataclasses.dataclass
class StringStatementConfiguration:
"""Configuration related to string statements."""
random_string_weight: float = 0.3
"""Probability use the random generator for string statement generation."""
faker_string_weight: float = 0.3
"""Probability use the Faker generator for string statement generation."""
fandango_string_weight: float = 0.4
"""Probability use the Fandango generator for string statement generation."""
fandango_faker_string_weight: float = 0.0
"""Probability use the Fandango + Faker generator for string statement generation."""
token_assembly_probability: float = 0.2
"""Probability to build a string by concatenating several seeded string constants.
Modules whose branches depend on token-structured inputs (``"two thousand"``,
``"2h32m"``, ``"P1Y2M10D"``) define their vocabulary as string literals, so the
tokens end up in the constant pool but are never combined. Assembling them lets
the search reach those branches. Only applies when at least two distinct string
constants were seeded."""
max_assembled_tokens: int = 4
"""Upper bound on the number of seeded tokens concatenated into one string."""
[docs]
@dataclasses.dataclass
class StoppingConfiguration:
"""Configuration related to when Pynguin should stop.
Note that these are mostly soft-limits rather than hard limits, because
the search algorithms only check the condition at the start of each algorithm
iteration.
"""
maximum_search_time: int = -1
"""Time (in seconds) that can be used for generating tests."""
maximum_test_executions: int = -1
"""Maximum number of test cases to be executed."""
maximum_statement_executions: int = -1
"""Maximum number of test cases to be executed."""
maximum_slicing_time: int = 600
"""Time budget (in seconds) that can be used for slicing."""
maximum_iterations: int = -1
"""Maximum iterations"""
maximum_test_execution_timeout: int = 5
"""The maximum time (in seconds) after which a test case times out."""
maximum_module_execution_timeout: int = 5
"""Maximum time (in seconds) allowed to import, reload, or exec() an SUT or
mutant module outside the test executor. Values <= 0 disable the limit."""
maximum_coverage: int = 100
"""The maximum percentage of coverage after which the generation shall stop."""
maximum_coverage_plateau: int = -1
"""Maximum number of algorithm iterations without coverage change before the
algorithms stops."""
minimum_coverage: int = 100
"""Minimum coverage for the plateau-based stopping condition. Expects values larger
than 0 but less than 100 to activate the stopping condition; also requires the
setting of minimum_plateau_iterations."""
minimum_plateau_iterations: int = -1
"""Minimum iterations without a coverage change to stop early. Expects values
larger than 0; also requires the setting of minimum_coverage."""
maximum_memory: int = 3000
"""Maximum memory usage in MB after which the generation shall stop.
Expects values in MB greater than 0 of -1 to disable the check."""
test_execution_time_per_statement: int = 1
"""The time (in seconds) per statement that a test is allowed to run
(up to maximum_test_execution_timeout)."""
[docs]
@dataclasses.dataclass
class LLMConfiguration:
"""Configuration for the LLM."""
api_key: str = ""
"""The api key to call OpenAI LLM with."""
model_name: str = "gpt-4o-mini"
"""The OpenAI Model to use for completions."""
llm_url: str = ""
"""Base URL for the LLM API endpoint (leave empty for default OpenAI)."""
hybrid_initial_population: bool = False
"""Whether to include the LLM test cases in the initial population (pre-search
seeding). Empirically, seeding spends token budget up front and delays the
search without improving final coverage; the cost-coverage optimum is to leave
this off and rely on ``call_llm_on_stall_detection`` (stagnation-triggered
querying) instead."""
llm_test_case_percentage: float = 0.5
"""The percentage of LLM test cases in on the initial population. The value must
be from [0.0, 1.0]."""
enable_response_caching: bool = False
"""Whether to enable caching for responses of the model."""
call_llm_for_uncovered_targets: bool = False
"""Whether to call the LLM for the uncovered targets once, before the search
iterations start. Like seeding, this front-loads token cost; prefer
``call_llm_on_stall_detection`` for the recommended stagnation-only setup."""
coverage_threshold: float = 1
"""The coverage threshold when to call the LLM for low-coverage targets.
The value must be from [0.0, 1.0]."""
call_llm_on_stall_detection: bool = False
"""Whether to query the LLM for uncovered targets when the search stalls
(stagnation-triggered querying). This is the recommended integration mode: keep
``hybrid_initial_population`` and ``call_llm_for_uncovered_targets`` off and
enable this so the GA covers easy goals first and the LLM is only spent to break
through plateaus."""
stall_detection_window_seconds: int = 30
"""Wall-clock seconds without a coverage gain before the search is considered
stalled and the LLM is queried (only used when ``call_llm_on_stall_detection`` is
enabled). Roughly 10%% of a standard 300s budget. Set to <= 0 to fall back to
the iteration-count based ``max_plateau_len`` heuristic instead."""
max_plateau_len: int = 25
"""The number of iterations without a coverage change before soliciting the LLM.
Only used as a fallback when ``stall_detection_window_seconds`` is <= 0; the
time-based window is preferred because iteration duration varies widely."""
max_llm_interventions: int = -1
"""The maximum number of allowed stall-triggered LLM interventions per run.
Defaults to ``-1`` (unlimited, bounded only by the search budget). This is safe
because interventions are naturally rate-limited by the stall-detection window
and the late-budget guard, so they fire at most once per detected plateau. Set
to a non-negative value to cap the number of interventions per run instead."""
min_remaining_budget_for_llm: int = 45
"""Late-budget guard: suppress stall-triggered LLM queries when fewer than this
many seconds of search time remain, to avoid spending tokens on a query that
cannot return and be integrated in time. Only applies when a maximum search time
is configured; set to <= 0 to disable the guard."""
max_context_chars: int = 64000
"""Maximum number of characters of module source code sent to the LLM. Coverage
gains saturate beyond this budget while token consumption keeps growing, so the
source is truncated to this length. Set to <= 0 to disable truncation."""
max_retries: int = 8
"""The maximum number of retries for LLM requests."""
request_timeout: float | None = 30.0
"""The timeout for LLM requests in seconds (``None`` disables the timeout). A
30s timeout keeps query latency low during active evolutionary search. Callers
that issue large batch requests outside the search (notably LLM refinement, see
``LLMRefinementConfiguration.request_timeout``) override this per request."""
max_request_time: float = 300.0
"""Total wall-clock budget (in seconds) for a single logical LLM request, covering
every retry attempt and the backoff waits between them. A retry is only started
when the budget can still accommodate it. Set to <= 0 to disable the bound."""
enable_thinking: bool | None = False
"""Whether to request reasoning ("thinking") output from a reasoning model, sent
to the API as ``extra_body={"chat_template_kwargs": {"enable_thinking": ...}}``.
Defaults to ``False``, suppressing reasoning output: reasoning models otherwise
spend the bulk of their tokens on a discarded reasoning trace, driving up latency
until requests exceed ``request_timeout``. Set to ``None`` to omit the field
entirely and leave the provider's default behaviour unchanged, e.g. for a
non-reasoning model or a provider that rejects unknown ``extra_body`` keys."""
cache_dir: str = "~/.cache/pynguin/llm"
"""The directory to store cached responses."""
max_concurrency: int = 5
"""The maximum number of concurrent LLM requests when using async mode."""
llm_mode: LLMMode = LLMMode.SYNC
"""The execution mode for LLM queries in search algorithms (sync or async)."""
[docs]
@dataclasses.dataclass
class MockGenerationConfiguration:
"""Configuration for mock generation."""
mock_generation_enabled: bool = False
"""Enable the mock generation pipeline. Requires proxy_cache_url or
PYNGUIN_PROXY_CACHE_URL to be set, otherwise falls back to default Pynguin."""
proxy_cache_url: str = ""
"""Base URL of the proxy-cache service.
Leave empty to fall back to the PYNGUIN_PROXY_CACHE_URL environment variable."""
base_rules_cache_id: str = ""
"""Optional proxy-cache identifier for pre-seeded rules from mock_rule_generator.
When provided, the dependency analyzer consults these rules before the LLM.
Omit to rely purely on the LLM classify endpoint."""
[docs]
@dataclasses.dataclass
class LocalSearchConfiguration:
"""Configurations for local search."""
local_search: bool = True
"""Defines if local search is generally enabled or not."""
local_search_same_datatype: bool = True
"""Defines if local search within same datatype is enabled or not"""
local_search_different_datatype: bool = False
"""Defines if local search with different datatypes is enabled or not"""
local_search_llm: bool = False
"""Defines if local search with llm is enabled or not"""
local_search_primitives: bool = True
"""Toggles if local search for primitive types is enabled or not"""
local_search_collections: bool = False
"""Toggles if local search for collection types is enabled or not"""
local_search_complex_objects: bool = False
"""Toggles if local search for complex object types is enabled or not"""
local_search_probability: float = 0.02
"""Probability of starting local search on the specific candidate"""
local_search_time: int = 5000
"""The budget which each local search iteration has in ms."""
ls_int_delta_increasing_factor: int = 2
"""The factor which defines how much the delta should increase for each iteration for integer
local search."""
ls_string_random_mutation_count: int = 10
"""The number of mutations made to the string to determine if it's worth to completely mutate
the string."""
ls_random_parametrized_statement_call_count: int = 10
"""The number of mutations made to a parametrized statement call to determine if it's worth to
further do local search"""
ls_max_different_type_mutations: int = 10
"""The max number of searches for different datatypes"""
ls_different_type_primitive_probability: float = 0.3
"""Weight of generating primitive statements when changing the statement type."""
ls_different_type_collection_probability: float = 0.3
"""Weight of generating collection statements when changing the statement type."""
ls_dict_max_insertions: int = 10
"""Max number of unsuccessful insertions of entries."""
ls_llm_whole_module: bool = False
"""Whether to provide the whole module or selected functions/methods to the LLM for context
when performing llm local search."""
[docs]
@dataclasses.dataclass
class ToCoverConfiguration:
"""Configuration of which code elements are included or excluded as coverage goals.
Pynguin instruments only the respective parts of the SUT module for coverage measurement.
All configuration options are resolved as SUT line numbers which must not overlap
('only' and 'no' at the same time).
"""
only_cover: list[str] = dataclasses.field(default_factory=list)
"""The list of space-separated qualified function, method or class names to only cover."""
only_cover_line_ranges: list[str] = dataclasses.field(default_factory=list)
"""The list of line numbers or line ranges to only cover, e.g. 10 or 10-20."""
no_cover: list[str] = dataclasses.field(default_factory=list)
"""The list of space-separated qualified function, method or class names to not cover.
Automatically include the methods of the `ignore_methods` argument."""
enable_inline_pynguin_no_cover: bool = True
"""Enable inline ``pynguin: no cover``."""
enable_inline_pragma_no_cover: bool = True
"""Enable inline ``pragma: no cover``."""
[docs]
@dataclasses.dataclass
class LLMRefinementConfiguration:
"""Configuration for LLM-based test refinement.
This enables post-processing of generated tests using a large language model
to improve readability and fix failing tests. The model name and API key
are taken from :class:`LLMConfiguration` (``large_language_model.model_name``
and ``large_language_model.api_key``) so that all LLM features share a single
provider setup.
"""
enabled: bool = False
"""Enable LLM-based test refinement."""
max_repair_iterations: int = 2
"""Maximum number of iterations to attempt test repair. Capping the multi-turn
syntax/type repair loop at 2 trades a tiny amount of test salvage for markedly
lower latency and token usage."""
max_tests: int | None = None
"""Maximum number of tests to refine (None = all tests)."""
request_timeout: float | None = 180.0
"""Per-attempt timeout (in seconds) for refinement LLM requests. Refinement is
batch post-processing whose module-level prompt carries the whole generated suite,
so it needs more time than the search-calibrated
``large_language_model.request_timeout``. ``None`` falls back to that value."""
save_original: bool = True
"""Save the original unrefined test file."""
save_refined: bool = True
"""Save the refined test file with _refined suffix."""
enable_dependency_context: bool = False
"""Enable retrieving dependent class/method signatures to include in prompt context."""
enable_usage_examples: bool = False
"""Enable retrieving usage examples from the codebase to include in prompt context."""
max_dependencies: int = 10
"""Maximum number of dependencies to collect context for."""
max_usage_examples: int = 3
"""Maximum number of usage examples to collect context for."""
enable_mutation_strengthening: bool = False
"""Enable mutation-driven assertion strengthening."""
max_mutation_iterations: int = 3
"""Maximum iterations for mutation-driven assertion strengthening."""
mutation_granularity: MutationStrengtheningGranularity = (
MutationStrengtheningGranularity.FULL_MODULE
)
"""Granularity of mutation-driven assertion strengthening (full_module or per_test).
Defaults to full_module."""
max_mutants_per_prompt: int = 15
"""Maximum number of surviving mutants to include in a single prompt batch.
Surviving mutants beyond this limit are chunked into multiple module-level requests."""
max_surviving_mutants: int = 50
"""Maximum number of mutants to generate and evaluate during module-level mutation
strengthening."""
refinement_granularity: RefinementGranularity = RefinementGranularity.COMBINED
"""Whether readability refinement and semantic-assertion generation are done with a
single combined module-level prompt (``combined``, default; most token-efficient,
highest-variance), two separate module-level prompts (``module_separate``), or one
prompt per test (``per_test``; previous behavior, most robust). Repair is always
per-broken-test. Any truncated or unparseable module-level response falls back
automatically to the per-test path."""
[docs]
@dataclasses.dataclass
class Configuration:
"""General configuration for the test generator."""
project_path: str
"""Path to the project the generator shall create tests for."""
module_name: str
"""Name of the module for which the generator shall create tests."""
test_case_output: TestCaseOutputConfiguration
"""Configuration for how test cases should be output."""
algorithm: Algorithm = Algorithm.DYNAMOSA
"""The algorithm that shall be used for generation."""
statistics_output: StatisticsOutputConfiguration = dataclasses.field(
default_factory=StatisticsOutputConfiguration
)
"""Statistic Output configuration."""
stopping: StoppingConfiguration = dataclasses.field(default_factory=StoppingConfiguration)
"""Stopping configuration."""
large_language_model: LLMConfiguration = dataclasses.field(default_factory=LLMConfiguration)
"""Large Language Model(LLM) configuration."""
mock_generation: MockGenerationConfiguration = dataclasses.field(
default_factory=MockGenerationConfiguration
)
"""Mock generation configuration."""
string_statement: StringStatementConfiguration = dataclasses.field(
default_factory=StringStatementConfiguration
)
"""String statement configuration."""
seeding: SeedingConfiguration = dataclasses.field(default_factory=SeedingConfiguration)
"""Seeding configuration."""
type_inference: TypeInferenceConfiguration = dataclasses.field(
default_factory=TypeInferenceConfiguration
)
"""Type inference configuration."""
pynguinml: PynguinMLConfiguration = dataclasses.field(default_factory=PynguinMLConfiguration)
"""PynguinML configuration."""
test_creation: TestCreationConfiguration = dataclasses.field(
default_factory=TestCreationConfiguration
)
"""Test creation configuration."""
generator_selection: GeneratorSelectionConfiguration = dataclasses.field(
default_factory=GeneratorSelectionConfiguration
)
search_algorithm: SearchAlgorithmConfiguration = dataclasses.field(
default_factory=SearchAlgorithmConfiguration
)
"""Search algorithm configuration."""
mio: MIOConfiguration = dataclasses.field(default_factory=MIOConfiguration)
"""Configuration used for the MIO algorithm."""
random: RandomConfiguration = dataclasses.field(default_factory=RandomConfiguration)
"""Configuration used for the RANDOM algorithm."""
island: IslandConfiguration = dataclasses.field(default_factory=IslandConfiguration)
"""Island-model parallel search configuration."""
llm_worker: LLMWorkerConfiguration = dataclasses.field(default_factory=LLMWorkerConfiguration)
"""Centralized asynchronous LLM query worker configuration."""
to_cover: ToCoverConfiguration = dataclasses.field(default_factory=ToCoverConfiguration)
"""Configuration of which code elements are included or excluded as coverage goals."""
llm_refinement: LLMRefinementConfiguration = dataclasses.field(
default_factory=LLMRefinementConfiguration
)
"""LLM-based test refinement configuration."""
ignore_modules: list[str] = dataclasses.field(default_factory=list)
"""Ignore the modules specified here from the module analysis."""
ignore_methods: list[str] = dataclasses.field(default_factory=list)
"""Ignore the methods specified here from the module analysis."""
element_visibility: ElementVisibility = ElementVisibility.PUBLIC
"""Which elements (by naming convention) of the module under test are included in
the test cluster: PUBLIC (default) only public elements, PROTECTED also elements
with a single leading underscore, ALL also private (double leading underscore,
including name-mangled) elements. Only applies to the module under test;
non-public elements of dependency modules are always excluded."""
subprocess: bool = False
"""Run the test generation in a subprocess."""
subprocess_if_recommended: bool = True
"""Use the subprocess mode if the SUT analysis recommends it based on used
C extension modules."""
local_search: LocalSearchConfiguration = dataclasses.field(
default_factory=LocalSearchConfiguration
)
"""Local search configuration."""
use_master_worker: bool = True
"""Use master-worker architecture for fault tolerance."""
filesystem_isolation: bool = False
"""Whether to use filesystem isolation during test execution.
Filesystem isolation provides some safety by tracking and controlling file operations,
e.g. preventing file deletion of non-created files, but adds performance overhead."""
# Singleton instance of the configuration.
configuration = Configuration(
project_path="",
module_name="",
test_case_output=TestCaseOutputConfiguration(output_path=""),
)