Differential D8684 Diff 40813 examples/docs_snippets/docs_snippets/concepts/solids_pipelines/solid_events.py

Changeset View

Standalone View

examples/docs_snippets/docs_snippets/concepts/solids_pipelines/solid_events.py

	Show First 20 Lines • Show All 77 Lines • ▼ Show 20 Lines

	@solid			@solid
	def my_metadata_output(context):			def my_metadata_output(context):
	df = get_some_data()			df = get_some_data()
	yield Output(			yield Output(
	df,			df,
	metadata={			metadata={
	"text_metadata": "Text-based metadata for this event",			"text_metadata": "Text-based metadata for this event",
	"dashboard_url": EventMetadata.url("http://mycoolsite.com/url_for_my_data"),			"dashboard_url": EventMetadata.url(
				"http://mycoolsite.com/url_for_my_data"
				),
	"raw_count": len(df),			"raw_count": len(df),
	"size (bytes)": calculate_bytes(df),			"size (bytes)": calculate_bytes(df),
	},			},
	)			)


	# end_solid_output_3			# end_solid_output_3

	# start_metadata_expectation_solid			# start_metadata_expectation_solid


	@solid			@solid
	def my_metadata_expectation_solid(context, df):			def my_metadata_expectation_solid(context, df):
	df = do_some_transform(df)			df = do_some_transform(df)
	yield ExpectationResult(			yield ExpectationResult(
	success=len(df) > 0,			success=len(df) > 0,
	description="ensure dataframe has rows",			description="ensure dataframe has rows",
	metadata={			metadata={
	"text_metadata": "Text-based metadata for this event",			"text_metadata": "Text-based metadata for this event",
	"dashboard_url": EventMetadata.url("http://mycoolsite.com/url_for_my_data"),			"dashboard_url": EventMetadata.url(
				"http://mycoolsite.com/url_for_my_data"
				),
	"raw_count": len(df),			"raw_count": len(df),
	"size (bytes)": calculate_bytes(df),			"size (bytes)": calculate_bytes(df),
	},			},
	)			)
	yield Output(df)			yield Output(df)


	# end_metadata_expectation_solid			# end_metadata_expectation_solid

	# start_failure_solid			# start_failure_solid


	@solid			@solid
	def my_failure_solid():			def my_failure_solid():
	path = "/path/to/files"			path = "/path/to/files"
	my_files = get_files(path)			my_files = get_files(path)
	if len(my_files) == 0:			if len(my_files) == 0:
	raise Failure(			raise Failure(
	description="No files to process",			description="No files to process",
	metadata={			metadata={
	"filepath": EventMetadata.path(path),			"filepath": EventMetadata.path(path),
	"dashboard_url": EventMetadata.url("http://mycoolsite.com/failures"),			"dashboard_url": EventMetadata.url(
				"http://mycoolsite.com/failures"
				),
	},			},
	)			)
	return some_calculation(my_files)			return some_calculation(my_files)


	# end_failure_solid			# end_failure_solid

	# start_failure_metadata_solid			# start_failure_metadata_solid


	@solid			@solid
	def my_failure_metadata_solid():			def my_failure_metadata_solid():
	path = "/path/to/files"			path = "/path/to/files"
	my_files = get_files(path)			my_files = get_files(path)
	if len(my_files) == 0:			if len(my_files) == 0:
	raise Failure(			raise Failure(
	description="No files to process",			description="No files to process",
	metadata={			metadata={
	"filepath": EventMetadata.path(path),			"filepath": EventMetadata.path(path),
	"dashboard_url": EventMetadata.url("http://mycoolsite.com/failures"),			"dashboard_url": EventMetadata.url(
				"http://mycoolsite.com/failures"
				),
	},			},
	)			)
	return some_calculation(my_files)			return some_calculation(my_files)


	# end_failure_metadata_solid			# end_failure_metadata_solid

	# start_retry_solid			# start_retry_solid
	Show All 30 Lines