Click here to go to the corresponding page for the latest version of DIALS
dials.integrate¶
Introduction¶
This program is used to integrate the reflections on the diffraction images. It is called with an experiment list outputted from dials.index or dials.refine and a corresponding set of strong spots from which a profile model is calculated. The program will output a set of integrated reflections and an experiment list with additional profile model data. The data can be reintegrated using the same profile model by inputting this integrated_experiments.json file back into dials.integate.
Examples:
dials.integrate experiments.json indexed.pickle
dials.integrate experiments.json indexed.pickle output.reflections=integrated.pickle
dials.integrate experiments.json indexed.pickle profile.fitting=False
dials.integrate experiments.json indexed.pickle background.algorithm=glm
Basic parameters¶
output {
experiments = 'integrated_experiments.json'
reflections = 'integrated.pickle'
phil = 'dials.integrate.phil'
log = 'dials.integrate.log'
debug_log = 'dials.integrate.debug.log'
report = None
include_bad_reference = False
}
scan_range = None
create_profile_model = True
exclude_images = None
verbosity = 1
integration {
lookup {
mask = None
}
block {
size = auto
units = *degrees radians frames
threshold = 0.99
force = False
max_memory_usage = 0.75
}
use_dynamic_mask = True
debug {
reference {
filename = "reference_profiles.pickle"
output = False
}
during = modelling *integration
output = False
separate_files = True
delete_shoeboxes = False
select = None
split_experiments = True
}
profile {
fitting = True
validation {
number_of_partitions = 1
min_partition_size = 100
}
}
overlaps_filter {
foreground_foreground {
enable = False
}
foreground_background {
enable = False
}
}
mp {
method = *none drmaa sge lsf pbs
njobs = 1
nproc = 1
}
summation {
detector_gain = 1
}
background {
algorithm = *glm gmodel simple null median
glm {
robust {
tuning_constant = 1.345
}
model {
algorithm = constant2d *constant3d loglinear2d loglinear3d
}
min_pixels = 10
}
gmodel {
robust {
algorithm = False
tuning_constant = 1.345
}
min_pixels = 10
model = None
}
simple {
outlier {
algorithm = *null nsigma truncated normal plane tukey
}
model {
algorithm = constant2d *constant3d linear2d linear3d
}
min_pixels = 10
}
median {
}
}
centroid {
algorithm = *simple
}
}
profile {
algorithm = *gaussian_rs
gaussian_rs {
scan_varying = False
min_spots {
overall = 50
per_degree = 20
}
sigma_m_algorithm = basic *extended
parameters {
sigma_b = None
sigma_m = None
}
filter {
min_zeta = 0.05
}
fitting {
scan_step = 5
grid_size = 5
threshold = 0.02
grid_method = single *regular_grid circular_grid spherical_grid
fit_method = *reciprocal_space detector_space
detector_space {
deconvolution = False
}
}
}
}
prediction {
d_min = None
d_max = None
margin = 1
force_static = False
padding = 1.0
}
significance_filter {
enable = False
d_min = None
n_bins = 20
isigi_cutoff = 1.0
}
absorption_correction {
apply = False
algorithm = fuller_kapton other
fuller_kapton {
xtal_height_above_kapton_mm {
value = 0.02
sigma = 0.01
}
rotation_angle_deg {
value = 1.15
sigma = 0.1
}
kapton_half_width_mm {
value = 1.5875
sigma = 0.5
}
kapton_thickness_mm {
value = 0.05
sigma = 0.005
}
smart_sigmas = False
within_spot_sigmas = True
}
}
Full parameter definitions¶
output {
experiments = 'integrated_experiments.json'
.help = "The experiments output filename"
.type = str
reflections = 'integrated.pickle'
.help = "The integrated output filename"
.type = str
phil = 'dials.integrate.phil'
.help = "The output phil file"
.type = str
log = 'dials.integrate.log'
.help = "The log filename"
.type = str
debug_log = 'dials.integrate.debug.log'
.help = "The debug log filename"
.type = str
report = None
.help = "The integration report filename (*.xml or *.json)"
.type = str
include_bad_reference = False
.help = "Include bad reference data including unindexed spots, and"
"reflections whose predictions are messed up in the reflection"
"table output. Reflections will have the 'bad_reference' flag set."
.type = bool
}
scan_range = None
.help = "Explicitly specify the images to be processed. Only applicable when"
"experiment list contains a single imageset."
.type = ints(size=2)
.multiple = True
create_profile_model = True
.help = "Create the profile model"
.type = bool
sampling
.expert_level = 1
{
reflections_per_degree = 50
.help = "The number of predicted reflections per degree of the sweep to"
"integrate."
.type = float(value_min=0, allow_none=True)
minimum_sample_size = 1000
.help = "cutoff that determines whether subsetting of the input "
"prediction list is done"
.type = int(allow_none=True)
maximum_sample_size = None
.help = "The maximum number of predictions to integrate. Overrides"
"reflections_per_degree if that produces a larger sample size."
.type = int(value_min=1, allow_none=True)
integrate_all_reflections = True
.help = "Override reflections_per_degree and integrate all predicted"
"reflections."
.type = bool
}
exclude_images = None
.help = "Exclude images from integration (e.g. 1,2,3,4,5 etc)"
.type = ints
verbosity = 1
.help = "The verbosity level"
.type = int(value_min=0, allow_none=True)
integration {
lookup
.help = "Parameters specifying lookup file path"
{
mask = None
.help = "The path to the mask file."
.type = str
}
block {
size = auto
.help = "The block size in rotation angle (degrees)."
.type = float(allow_none=True)
units = *degrees radians frames
.help = "The units of the block size"
.type = choice
threshold = 0.99
.help = "For block size auto the block size is calculated by sorting"
"reflections by the number of frames they cover and then"
"selecting the block size to be 2*nframes[threshold] such that"
"100*threshold % of reflections are guarenteed to be fully"
"contained in 1 block"
.type = float(value_min=0, value_max=1, allow_none=True)
force = False
.help = "If the number of processors is 1 and force is False, then the"
"number of blocks may be set to 1. If force is True then the"
"block size is always calculated."
.type = bool
max_memory_usage = 0.75
.help = "The maximum percentage of total physical memory to use for"
"allocating shoebox arrays."
.type = float(value_min=0, value_max=1, allow_none=True)
}
use_dynamic_mask = True
.help = "Use dynamic mask if available"
.type = bool
debug {
reference {
filename = "reference_profiles.pickle"
.help = "The filename for the reference profiles"
.type = str
output = False
.help = "Save the reference profiles"
.type = bool
}
during = modelling *integration
.help = "Do debugging during modelling or integration"
.type = choice
output = False
.help = "Save shoeboxes after each processing task."
.type = bool
separate_files = True
.help = "If this is true, the shoeboxes are saved in separate files from"
"the output integrated.pickle file. This is necessary in most"
"cases since the amount of memory used by the shoeboxes is"
"typically greater than the available system memory. If,"
"however, you know that memory is not an issue, you can saved"
"the shoeboxes in the integrated.pickle file by setting this"
"option to False. This only works if the debug output is during"
"integrated and not modelling."
.type = bool
delete_shoeboxes = False
.help = "Delete shoeboxes immediately before saving files. This option"
"in combination with debug.output=True enables intermediate"
"processing steps to make use of shoeboxes."
.type = bool
select = None
.help = "A string specifying the selection. The string should be of the"
"form: select=${COLUMN}[<|<=|==|!=|>=|>]${VALUE}. In addition to"
"the items in the reflection table, the following implicit"
"columns are defined if the necessary data is there: "
"intensity.sum.i_over_sigma intensity.prf.i_over_sigma"
.type = reflection_table_selector
split_experiments = True
.help = "Split shoeboxes into different files"
.type = bool
}
profile {
fitting = True
.help = "Use profile fitting if available"
.type = bool
validation {
number_of_partitions = 1
.help = "The number of subsamples to take from the reference spots. If"
"the value is 1, then no validation is performed."
.type = int(value_min=1, allow_none=True)
min_partition_size = 100
.help = "The minimum number of spots to use in each subsample."
.type = int(value_min=1, allow_none=True)
}
}
filter
.expert_level = 1
{
min_zeta = 0.05
.help = "Filter the reflections by the value of zeta. A value of less"
"than or equal to zero indicates that this will not be used. A"
"positive value is used as the minimum permissable value."
.type = float(value_min=0, value_max=1, allow_none=True)
max_shoebox_overlap = 1.0
.help = "Filter reflections whose shoeboxes are overlapped by greater"
"than the requested amount. Note that this is not the percentage"
"of the peak that is overlapped but rather the percentage of the"
"shoebox (background and foreground). This can be useful when"
"the detector is too close and many overlapping reflections are"
"predicted at high resolution causing memory issues."
.type = float(value_min=0, value_max=1, allow_none=True)
ice_rings = False
.help = "Set the ice ring flags"
.type = bool
}
overlaps_filter {
foreground_foreground {
enable = False
.help = "Remove all spots in which neighbors' foreground impinges on"
"the spot's foreground"
.type = bool
}
foreground_background {
enable = False
.help = "Remove all spots in which neighbors' foreground impinges on"
"the spot's background"
.type = bool
}
}
mp {
method = *none drmaa sge lsf pbs
.help = "The cluster method to use"
.type = choice
njobs = 1
.help = "The number of cluster jobs to use"
.type = int(value_min=1, allow_none=True)
nproc = 1
.help = "The number of processes to use per cluster job"
.type = int(value_min=1, allow_none=True)
}
summation {
detector_gain = 1
.help = "Multiplier for variances after integration of still images. See"
"Leslie 1999."
.type = float(allow_none=True)
}
background
.help = " E x t e n s i o n s f o r b a c k g r o u n d a l g o r i"
"t h m s . "
{
algorithm = *glm gmodel simple null median
.help = "The choice of algorithm"
.type = choice
glm
.help = "An extension class implementing a robust GLM background"
"algorithm."
{
robust {
tuning_constant = 1.345
.help = "The tuning constant for robust estimation"
.type = float(allow_none=True)
}
model {
algorithm = constant2d *constant3d loglinear2d loglinear3d
.help = "The background model to fit"
.type = choice
}
min_pixels = 10
.help = "The minimum number of pixels required"
.type = int(value_min=1, allow_none=True)
}
gmodel
.help = "An extension class implementing a robust GLM background"
"algorithm."
{
robust {
algorithm = False
.help = "Use the robust algorithm"
.type = bool
tuning_constant = 1.345
.help = "The tuning constant for robust estimation"
.type = float(allow_none=True)
}
min_pixels = 10
.help = "The minimum number of pixels required"
.type = int(value_min=1, allow_none=True)
model = None
.help = "The model filename"
.type = str
}
simple
.help = "An extension class implementing simple background subtraction."
{
outlier
.help = "Outlier rejection prior to background fit"
{
algorithm = *null nsigma truncated normal plane tukey
.help = "The outlier rejection algorithm."
.type = choice
nsigma
.help = "Parameters for nsigma outlier rejector"
.expert_level = 1
{
lower = 3
.help = "Lower n sigma"
.type = float(allow_none=True)
upper = 3
.help = "Upper n sigma"
.type = float(allow_none=True)
}
truncated
.help = "Parameters for truncated outlier rejector"
.expert_level = 1
{
lower = 0.01
.help = "Lower bound"
.type = float(allow_none=True)
upper = 0.01
.help = "Upper bound"
.type = float(allow_none=True)
}
normal
.help = "Parameters for normal outlier rejector"
.expert_level = 1
{
min_pixels = 10
.help = "The minimum number of pixels to use in calculating the"
"background intensity."
.type = int(allow_none=True)
}
plane
.help = "Parameters for mosflm-like outlier rejector. This algorithm"
"is mainly used in conjunction with a linear 2d background."
.expert_level = 1
{
fraction = 1.0
.help = "The fraction of pixels to use in determining the initial"
"plane used for outlier rejection."
.type = float(allow_none=True)
n_sigma = 4.0
.help = "The number of standard deviations above the threshold"
"plane to use in rejecting outliers from background"
"calculation."
.type = float(allow_none=True)
}
tukey
.help = "Parameters for tukey outlier rejector"
.expert_level = 1
{
lower = 1.5
.help = "Lower IQR multiplier"
.type = float(allow_none=True)
upper = 1.5
.help = "Upper IQR multiplier"
.type = float(allow_none=True)
}
}
model
.help = "Background model"
{
algorithm = constant2d *constant3d linear2d linear3d
.help = "The choice of background model"
.type = choice
}
min_pixels = 10
.help = "The minimum number of pixels to compute the background"
.type = int(value_min=1, allow_none=True)
}
median
.help = "An extension class implementing a median background algorithm."
{
}
}
centroid
.help = " E x t e n s i o n s f o r c e n t r o i d a l g o r i t h"
"m s . "
{
algorithm = *simple
.help = "The choice of algorithm"
.type = choice
}
}
profile
.help = "
T h e i n t e r f a c e d e f i n i t i o n f o r "
"a p r o f i l e m o d e l .
"
{
algorithm = *gaussian_rs
.help = "The choice of algorithm"
.type = choice
gaussian_rs
.help = "An extension class implementing a reciprocal space gaussian"
"profile model."
{
scan_varying = False
.help = "Calculate a scan varying model"
.type = bool
min_spots
.help = "if (total_reflections > overall or reflections_per_degree >"
"per_degree) then do the profile modelling."
{
overall = 50
.help = "The minimum number of spots needed to do the profile"
"modelling"
.type = int(value_min=0, allow_none=True)
per_degree = 20
.help = "The minimum number of spots needed to do the profile"
"modelling"
.type = int(value_min=0, allow_none=True)
}
sigma_m_algorithm = basic *extended
.help = "The algorithm to compute mosaicity"
.type = choice
parameters {
sigma_b = None
.help = "Override the sigma_b value (degrees)"
.type = float(value_min=0, allow_none=True)
sigma_m = None
.help = "Override the sigma_m value (degrees)"
.type = float(value_min=0, allow_none=True)
}
filter {
min_zeta = 0.05
.help = "Filter reflections by min zeta"
.type = float(allow_none=True)
}
fitting {
scan_step = 5
.help = "Space between profiles in degrees"
.type = float(allow_none=True)
grid_size = 5
.help = "The size of the profile grid."
.type = int(allow_none=True)
threshold = 0.02
.help = "The threshold to use in reference profile"
.type = float(allow_none=True)
grid_method = single *regular_grid circular_grid spherical_grid
.help = "Select the profile grid method"
.type = choice
fit_method = *reciprocal_space detector_space
.help = "The fitting method"
.type = choice
detector_space {
deconvolution = False
.help = "Do deconvolution in detector space"
.type = bool
}
}
}
}
prediction {
d_min = None
.help = "The maximum resolution limit"
.type = float(allow_none=True)
d_max = None
.help = "The minimum resolution limit"
.type = float(allow_none=True)
margin = 1
.help = "The margin to use to scan varying prediction"
.type = int(allow_none=True)
force_static = False
.help = "For scan-varying prediction force scan-static prediction"
.type = bool
padding = 1.0
.help = "The padding in degrees"
.type = float(value_min=0, allow_none=True)
}
significance_filter {
enable = False
.help = "If enabled, the significance filter will, for every experiment,"
"find the highest resolution where the I/sigI remains above a"
"certain point (controlled by isigi_cutoff)."
.type = bool
d_min = None
.help = "High resolution cutoff for binning. If None, use the highest"
"resolution reflection as d_min."
.type = float(allow_none=True)
n_bins = 20
.help = "Number of bins to use when examining resolution falloff"
.type = int(allow_none=True)
isigi_cutoff = 1.0
.help = "I/sigI cutoff. Reflections in and past the first bin that falls"
"below this cutoff will not be retained"
.type = float(allow_none=True)
}
absorption_correction
.multiple = True
{
apply = False
.help = "must be supplied as a user-defined function with a specific"
"interface (not documented)"
.type = bool
algorithm = fuller_kapton other
.help = "a specific absorption correction, or implementation thereof"
.type = choice
fuller_kapton {
xtal_height_above_kapton_mm {
value = 0.02
.help = "height of the beam (or the irradiated crystal) above the"
"kapton tape"
.type = float(allow_none=True)
sigma = 0.01
.type = float(allow_none=True)
}
rotation_angle_deg {
value = 1.15
.help = "angle of the tape from vertical"
.type = float(allow_none=True)
sigma = 0.1
.type = float(allow_none=True)
}
kapton_half_width_mm {
value = 1.5875
.help = "forward distance from irradiated crystal to edge of tape"
"nearest detector"
.type = float(allow_none=True)
sigma = 0.5
.type = float(allow_none=True)
}
kapton_thickness_mm {
value = 0.05
.help = "tape thickness"
.type = float(allow_none=True)
sigma = 0.005
.type = float(allow_none=True)
}
smart_sigmas = False
.help = "apply spot-specific sigma corrections using kapton param sigmas"
.type = bool
within_spot_sigmas = True
.help = "turn this off to get a major speed-up"
.type = bool
}
}
