Plugin Cloudquery

Sync ​Sync

Table of Contents
yaml
type: "io.kestra.plugin.cloudquery.sync"
yaml
id: cloudquery_sync
namespace: company.team

tasks:
  - id: hn_to_duckdb
    type: io.kestra.plugin.cloudquery.Sync
    env:
      CLOUDQUERY_API_KEY: "{{ secret('CLOUDQUERY_API_KEY') }}"
    incremental: false
    configs:
      - kind: source
        spec:
          name: hackernews
          path: cloudquery/hackernews
          version: v3.0.13
          tables: ["*"]
          destinations: ["duckdb"]
          spec:
            item_concurrency: 100
            start_time: "{{ now() | dateAdd(-1, 'DAYS') }}"
      - kind: destination
        spec:
          name: duckdb
          path: cloudquery/duckdb
          version: v4.2.10
          write_mode: overwrite-delete-stale
          spec:
            connection_string: hn.db
yaml
id: cloudquery_sync
namespace: company.team

tasks:
  - id: hn_to_duckdb
    type: io.kestra.plugin.cloudquery.Sync
    incremental: false
    env:
        AWS_ACCESS_KEY_ID: "{{ secret('AWS_ACCESS_KEY_ID') }}"
        AWS_SECRET_ACCESS_KEY: "{{ secret('AWS_SECRET_ACCESS_KEY') }}"
        AWS_DEFAULT_REGION: "{{ secret('AWS_DEFAULT_REGION') }}"
        CLOUDQUERY_API_KEY: "{{ secret('CLOUDQUERY_API_KEY') }}"
        PG_CONNECTION_STRING: "postgresql://postgres:{{ secret('DB_PASSWORD') }}@host.docker.internal:5432/demo?sslmode=disable"
    configs:
      - sources.yml
      - destination.yml
Properties
Default ghcr.io/cloudquery/cloudquery:latest
SubType string
Default false
Default true
SubType string
Default 0
SubType string
Default busybox
Default true
SubType string
Default false
Default OVERWRITE
Possible Values
OVERWRITEFAILWARNIGNORE
SubType string
SubType string
Default ["{{flow.namespace}}"]
Default true
Default { "image": "busybox" }
Default default
Default ALWAYS
Possible Values
IF_NOT_PRESENTALWAYSNEVER
Default true
Validation RegExp \d+\.\d+\.\d+(-[a-zA-Z0-9-]+)?|([a-zA-Z0-9]+)
Default PT5S
Format duration
Default PT1H
Format duration
Default PT10M
Format duration
Default PT5S
Format duration
Default true
Default true
Default false
Validation RegExp \d+\.\d+\.\d+(-[a-zA-Z0-9-]+)?|([a-zA-Z0-9]+)
Default PT1H
Format duration
SubType integer
Default PT5S
Format duration
Default true
Default true
SubType string
Default ["https://www.googleapis.com/auth/cloud-platform"]
Validation RegExp \d+\.\d+\.\d+(-[a-zA-Z0-9-]+)?|([a-zA-Z0-9]+)
Default PT5S
Format duration
Default PT1H
Format duration
Possible Values
ACTION_UNSPECIFIEDRETRY_TASKFAIL_TASKUNRECOGNIZED
Default v1
Default RSA
Default https://kubernetes.default.svc
Validation RegExp \d+\.\d+\.\d+(-[a-zA-Z0-9-]+)?|([a-zA-Z0-9]+)
Default PT5S
Format duration
Default true
SubType string
Default e2-medium
Default 2
Minimum >= 0
Maximum <= 10
Default true
SubType string
Default ["https://www.googleapis.com/auth/cloud-platform"]
Validation RegExp \d+\.\d+\.\d+(-[a-zA-Z0-9-]+)?|([a-zA-Z0-9]+)
Default PT5S
Format duration
Default PT1H
Format duration
Min length 1
SubType
SubType string
SubType string
Default IF_NOT_PRESENT
Possible Values
IF_NOT_PRESENTALWAYSNEVER
SubType string
Default true
SubType
SubType string
Default [ "" ]
SubType string
Default VOLUME
Possible Values
MOUNTVOLUME
Default PT0S
Format duration
SubType string
Default IF_NOT_PRESENT
Possible Values
IF_NOT_PRESENTALWAYSNEVER
Validation RegExp \d+\.\d+\.\d+(-[a-zA-Z0-9-]+)?|([a-zA-Z0-9]+)
SubType string
Default true
SubType array
SubType string
SubType string
Default PT5S
Format duration
Default true
Default { "request": { "memory": "2048", "cpu": "1" } }
Default true
Default PT15M
Format duration
Validation RegExp \d+\.\d+\.\d+(-[a-zA-Z0-9-]+)?|([a-zA-Z0-9]+)
Default PT1H
Format duration