Page MenuHomePhabricator

model input selection example
ActivePublic

Authored by alangenfeld on Oct 28 2019, 3:45 PM.
def test_toggle_pipeline():
@lambda_solid
def generate_a():
return ['A']
@lambda_solid
def generate_b():
return ['B']
@lambda_solid
def concat(a, b):
return a + b
# To mirror the approach you had, we introduce a "mux" solid that
# is responsible for choosing which input to forward based on config.
@solid(config={'selected': Field(String)})
def input_mux(context, a, b, merged):
selected = context.solid_config['selected']
if selected == 'a':
return a
elif selected == 'b':
return b
else:
return merged
def map_to_mux(_ctx, cfg):
return {'input_mux': {'config': {'selected': cfg['selected']}}}
# We wrap all the input generation and selection in a composite to attempt to hide this complexity.
@composite_solid(
# We use the config mapping ability to make it so we dont have to provide
# config for a nested solid. This allows the composite to actually "hide"
# complexity of the choosing process.
config={'selected': Field(String)},
config_fn=map_to_mux,
)
def generate_input():
a = generate_a()
b = generate_b()
return input_mux(a, b, concat(a, b))
@lambda_solid
def do_stuff(selected_in):
return len(selected_in)
@pipeline
def selective_pipeline():
# A simpler example but you get the gist
do_stuff(generate_input())
# execute a few different selections
execute_pipeline(
selective_pipeline,
environment_dict={'solids': {'generate_input': {'config': {'selected': 'a'}}}},
)
execute_pipeline(
selective_pipeline,
environment_dict={'solids': {'generate_input': {'config': {'selected': 'b'}}}},
)

Event Timeline

alangenfeld created this object with visibility "Public (No Login Required)".
alangenfeld edited the content of this paste. (Show Details)
alangenfeld edited the content of this paste. (Show Details)