Page MenuHomeElementl
Differential D8684 Diff 40976 docs/content/concepts/configuration/config-schema.mdx

Changeset View

Standalone View

View Options

docs/content/concepts/configuration/config-schema.mdx

Show First 20 Lines • Show All 49 Lines • ▼ Show 20 Lines
When executing a pipeline with <PyObject module="dagster" object="execute_pipeline" />, you can specify the config values through `run_config` argument: When executing a pipeline with <PyObject module="dagster" object="execute_pipeline" />, you can specify the config values through `run_config` argument:
```python file=/concepts/configuration/execute_with_config.py startafter=start_execute_with_config endbefore=end_execute_with_config ```python file=/concepts/configuration/execute_with_config.py startafter=start_execute_with_config endbefore=end_execute_with_config
from dagster import execute_pipeline from dagster import execute_pipeline
execute_pipeline( execute_pipeline(
config_example_pipeline, config_example_pipeline,
run_config={"solids": {"config_example_solid": {"config": {"iterations": 1}}}}, run_config={
"solids": {"config_example_solid": {"config": {"iterations": 1}}}
},
) )
``` ```
### Dagster CLI ### Dagster CLI
When executing a pipeline from the command line, the easiest way to provide config is to put it into a YAML file, like: When executing a pipeline from the command line, the easiest way to provide config is to put it into a YAML file, like:
```YAML file=/concepts/configuration/good.yaml ```YAML file=/concepts/configuration/good.yaml
▲ Show 20 Lines • Show All 54 Lines • ▼ Show 20 Lines
```python file=/concepts/configuration/execute_with_config.py startafter=start_execute_with_bad_config endbefore=end_execute_with_bad_config ```python file=/concepts/configuration/execute_with_config.py startafter=start_execute_with_bad_config endbefore=end_execute_with_bad_config
from dagster import execute_pipeline from dagster import execute_pipeline
execute_pipeline( execute_pipeline(
configurable_pipeline_with_schema, configurable_pipeline_with_schema,
run_config={ run_config={
"solids": { "solids": {
"configurable_solid_with_schema": {"config": {"nonexistent_config_value": 1}} "configurable_solid_with_schema": {
"config": {"nonexistent_config_value": 1}
}
} }
}, },
) )
``` ```
The config editor in Dagit the page comes with typeaheads, schema validation, and schema documentation. You can also click the "Scaffold Missing Config" button to generate dummy values based on the config schema. The config editor in Dagit the page comes with typeaheads, schema validation, and schema documentation. You can also click the "Scaffold Missing Config" button to generate dummy values based on the config schema.
## Examples ## Examples
Show All 20 Lines
### Passing Configuration to Multiple Solids in a Pipeline ### Passing Configuration to Multiple Solids in a Pipeline
If you want multiple solids to share values, You can use <PyObject module="dagster" object="make_values_resource" /> to pass the values via a resource and reference that resource from any solid that wants to use it. If you want multiple solids to share values, You can use <PyObject module="dagster" object="make_values_resource" /> to pass the values via a resource and reference that resource from any solid that wants to use it.
It defaults to <PyObject module="dagster" object="Any" /> type, meaning Dagster will accept any config value provided for the resource: It defaults to <PyObject module="dagster" object="Any" /> type, meaning Dagster will accept any config value provided for the resource:
```python file=/concepts/configuration/make_values_resource_any.py ```python file=/concepts/configuration/make_values_resource_any.py
from dagster import ModeDefinition, execute_pipeline, make_values_resource, pipeline, solid from dagster import (
ModeDefinition,
execute_pipeline,
make_values_resource,
pipeline,
solid,
)
@solid(required_resource_keys={"value"}) @solid(required_resource_keys={"value"})
def solid1(context): def solid1(context):
context.log.info(f"value: {context.resources.value}") context.log.info(f"value: {context.resources.value}")
@solid(required_resource_keys={"value"}) @solid(required_resource_keys={"value"})
def solid2(context): def solid2(context):
context.log.info(f"value: {context.resources.value}") context.log.info(f"value: {context.resources.value}")
@pipeline(mode_defs=[ModeDefinition(resource_defs={"value": make_values_resource()})]) @pipeline(
mode_defs=[
ModeDefinition(resource_defs={"value": make_values_resource()})
]
)
def my_pipeline(): def my_pipeline():
solid1() solid1()
solid2() solid2()
execute_pipeline(my_pipeline, run_config={"resources": {"value": {"config": "some_value"}}}) execute_pipeline(
my_pipeline, run_config={"resources": {"value": {"config": "some_value"}}}
)
``` ```
You can also specify the schema of the values like: You can also specify the schema of the values like:
```python file=/concepts/configuration/make_values_resource_config_schema.py ```python file=/concepts/configuration/make_values_resource_config_schema.py
from dagster import ModeDefinition, execute_pipeline, make_values_resource, pipeline, solid from dagster import (
ModeDefinition,
execute_pipeline,
make_values_resource,
pipeline,
solid,
)
@solid(required_resource_keys={"values"}) @solid(required_resource_keys={"values"})
def solid1(context): def solid1(context):
context.log.info(f"my str: {context.resources.values['my_str']}") context.log.info(f"my str: {context.resources.values['my_str']}")
@solid(required_resource_keys={"values"}) @solid(required_resource_keys={"values"})
def solid2(context): def solid2(context):
context.log.info(f"my int: {context.resources.values['my_int']}") context.log.info(f"my int: {context.resources.values['my_int']}")
@pipeline( @pipeline(
mode_defs=[ mode_defs=[
ModeDefinition(resource_defs={"values": make_values_resource(my_str=str, my_int=int)}) ModeDefinition(
resource_defs={
"values": make_values_resource(my_str=str, my_int=int)
}
)
] ]
) )
def my_pipeline(): def my_pipeline():
solid1() solid1()
solid2() solid2()
execute_pipeline( execute_pipeline(
my_pipeline, run_config={"resources": {"values": {"config": {"my_str": "foo", "my_int": 1}}}} my_pipeline,
run_config={
"resources": {"values": {"config": {"my_str": "foo", "my_int": 1}}}
},
) )
``` ```
And pass the values via a run config like: And pass the values via a run config like:
```YAML file=/concepts/configuration/make_values_resource_values.yaml ```YAML file=/concepts/configuration/make_values_resource_values.yaml
resources: resources:
values: values:
config: config:
my_str: foo my_str: foo
my_int: 1 my_int: 1
``` ```