PySparkSubmit
yaml
type: "io.kestra.plugin.gcp.dataproc.batches.pysparksubmit"Examples
yaml
id: gcp_dataproc_py_spark_submit
namespace: company.team
tasks:
- id: py_spark_submit
type: io.kestra.plugin.gcp.dataproc.batches.PySparkSubmit
mainPythonFileUri: 'gs://spark-jobs-kestra/pi.py'
name: test-pyspark
region: europe-west3
Properties
mainPythonFileUri *Requiredstring
name *Requiredstring
region *Requiredstring
archiveUris array
SubType string
args array
SubType string
execution AbstractBatch-ExecutionConfiguration
fileUris array
SubType string
impersonatedServiceAccount string
jarFileUris array
SubType string
peripherals AbstractBatch-PeripheralsConfiguration
projectId string
scopes array
SubType string
Default
["https://www.googleapis.com/auth/cloud-platform"]serviceAccount string
Outputs
state string
Possible Values
STATE_UNSPECIFIEDPENDINGRUNNINGCANCELLINGCANCELLEDSUCCEEDEDFAILEDUNRECOGNIZEDDefinitions
io.kestra.plugin.gcp.dataproc.batches.AbstractBatch-PeripheralsConfiguration
metastoreService string
sparkHistoryServer AbstractBatch-SparkHistoryServerConfiguration
io.kestra.plugin.gcp.dataproc.batches.AbstractBatch-RuntimeConfiguration
containerImage string
properties object
SubType string
version string
io.kestra.plugin.gcp.dataproc.batches.AbstractBatch-SparkHistoryServerConfiguration
dataprocCluster string
io.kestra.plugin.gcp.dataproc.batches.AbstractBatch-ExecutionConfiguration
kmsKey string
networkTags array
SubType string