import * as pulumi from "@pulumi/pulumi"; import * as inputs from "../types/input"; import * as outputs from "../types/output"; /** * A resource representing streaming data from a source to a destination. * * To get more information about Stream, see: * * * [API documentation](https://cloud.google.com/datastream/docs/reference/rest/v1/projects.locations.streams) * * How-to Guides * * [Official Documentation](https://cloud.google.com/datastream/docs/create-a-stream) * * ## Example Usage * * ### Datastream Stream Full * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * import * as random from "@pulumi/random"; * * const project = gcp.organizations.getProject({}); * const instance = new gcp.sql.DatabaseInstance("instance", { * name: "my-instance", * databaseVersion: "MYSQL_8_0", * region: "us-central1", * settings: { * tier: "db-f1-micro", * backupConfiguration: { * enabled: true, * binaryLogEnabled: true, * }, * ipConfiguration: { * authorizedNetworks: [ * { * value: "34.71.242.81", * }, * { * value: "34.72.28.29", * }, * { * value: "34.67.6.157", * }, * { * value: "34.67.234.134", * }, * { * value: "34.72.239.218", * }, * ], * }, * }, * deletionProtection: true, * }); * const db = new gcp.sql.Database("db", { * instance: instance.name, * name: "db", * }); * const pwd = new random.index.Password("pwd", { * length: 16, * special: false, * }); * const user = new gcp.sql.User("user", { * name: "user", * instance: instance.name, * host: "%", * password: pwd.result, * }); * const sourceConnectionProfile = new gcp.datastream.ConnectionProfile("source_connection_profile", { * displayName: "Source connection profile", * location: "us-central1", * connectionProfileId: "source-profile", * mysqlProfile: { * hostname: instance.publicIpAddress, * username: user.name, * password: user.password, * }, * }); * const bucket = new gcp.storage.Bucket("bucket", { * name: "my-bucket", * location: "US", * uniformBucketLevelAccess: true, * }); * const viewer = new gcp.storage.BucketIAMMember("viewer", { * bucket: bucket.name, * role: "roles/storage.objectViewer", * member: project.then(project => `serviceAccount:service-${project.number}@gcp-sa-datastream.iam.gserviceaccount.com`), * }); * const creator = new gcp.storage.BucketIAMMember("creator", { * bucket: bucket.name, * role: "roles/storage.objectCreator", * member: project.then(project => `serviceAccount:service-${project.number}@gcp-sa-datastream.iam.gserviceaccount.com`), * }); * const reader = new gcp.storage.BucketIAMMember("reader", { * bucket: bucket.name, * role: "roles/storage.legacyBucketReader", * member: project.then(project => `serviceAccount:service-${project.number}@gcp-sa-datastream.iam.gserviceaccount.com`), * }); * const keyUser = new gcp.kms.CryptoKeyIAMMember("key_user", { * cryptoKeyId: "kms-name", * role: "roles/cloudkms.cryptoKeyEncrypterDecrypter", * member: project.then(project => `serviceAccount:service-${project.number}@gcp-sa-datastream.iam.gserviceaccount.com`), * }); * const destinationConnectionProfile = new gcp.datastream.ConnectionProfile("destination_connection_profile", { * displayName: "Connection profile", * location: "us-central1", * connectionProfileId: "destination-profile", * gcsProfile: { * bucket: bucket.name, * rootPath: "/path", * }, * }); * const _default = new gcp.datastream.Stream("default", { * streamId: "my-stream", * desiredState: "NOT_STARTED", * location: "us-central1", * displayName: "my stream", * labels: { * key: "value", * }, * sourceConfig: { * sourceConnectionProfile: sourceConnectionProfile.id, * mysqlSourceConfig: { * includeObjects: { * mysqlDatabases: [{ * database: "my-database", * mysqlTables: [ * { * table: "includedTable", * mysqlColumns: [{ * column: "includedColumn", * dataType: "VARCHAR", * collation: "utf8mb4", * primaryKey: false, * nullable: false, * ordinalPosition: 0, * }], * }, * { * table: "includedTable_2", * }, * ], * }], * }, * excludeObjects: { * mysqlDatabases: [{ * database: "my-database", * mysqlTables: [{ * table: "excludedTable", * mysqlColumns: [{ * column: "excludedColumn", * dataType: "VARCHAR", * collation: "utf8mb4", * primaryKey: false, * nullable: false, * ordinalPosition: 0, * }], * }], * }], * }, * maxConcurrentCdcTasks: 5, * }, * }, * destinationConfig: { * destinationConnectionProfile: destinationConnectionProfile.id, * gcsDestinationConfig: { * path: "mydata", * fileRotationMb: 200, * fileRotationInterval: "60s", * jsonFileFormat: { * schemaFileFormat: "NO_SCHEMA_FILE", * compression: "GZIP", * }, * }, * }, * backfillAll: { * mysqlExcludedObjects: { * mysqlDatabases: [{ * database: "my-database", * mysqlTables: [{ * table: "excludedTable", * mysqlColumns: [{ * column: "excludedColumn", * dataType: "VARCHAR", * collation: "utf8mb4", * primaryKey: false, * nullable: false, * ordinalPosition: 0, * }], * }], * }], * }, * }, * customerManagedEncryptionKey: "kms-name", * }, { * dependsOn: [keyUser], * }); * ``` * ### Datastream Stream Postgresql * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const source = new gcp.datastream.ConnectionProfile("source", { * displayName: "Postgresql Source", * location: "us-central1", * connectionProfileId: "source-profile", * postgresqlProfile: { * hostname: "hostname", * port: 5432, * username: "user", * password: "pass", * database: "postgres", * }, * }); * const destination = new gcp.datastream.ConnectionProfile("destination", { * displayName: "BigQuery Destination", * location: "us-central1", * connectionProfileId: "destination-profile", * bigqueryProfile: {}, * }); * const _default = new gcp.datastream.Stream("default", { * displayName: "Postgres to BigQuery", * location: "us-central1", * streamId: "my-stream", * desiredState: "RUNNING", * sourceConfig: { * sourceConnectionProfile: source.id, * postgresqlSourceConfig: { * maxConcurrentBackfillTasks: 12, * publication: "publication", * replicationSlot: "replication_slot", * includeObjects: { * postgresqlSchemas: [{ * schema: "schema", * postgresqlTables: [{ * table: "table", * postgresqlColumns: [{ * column: "column", * }], * }], * }], * }, * excludeObjects: { * postgresqlSchemas: [{ * schema: "schema", * postgresqlTables: [{ * table: "table", * postgresqlColumns: [{ * column: "column", * }], * }], * }], * }, * }, * }, * destinationConfig: { * destinationConnectionProfile: destination.id, * bigqueryDestinationConfig: { * dataFreshness: "900s", * sourceHierarchyDatasets: { * datasetTemplate: { * location: "us-central1", * }, * }, * }, * }, * backfillAll: { * postgresqlExcludedObjects: { * postgresqlSchemas: [{ * schema: "schema", * postgresqlTables: [{ * table: "table", * postgresqlColumns: [{ * column: "column", * }], * }], * }], * }, * }, * }); * ``` * ### Datastream Stream Oracle * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const source = new gcp.datastream.ConnectionProfile("source", { * displayName: "Oracle Source", * location: "us-central1", * connectionProfileId: "source-profile", * oracleProfile: { * hostname: "hostname", * port: 1521, * username: "user", * password: "pass", * databaseService: "ORCL", * }, * }); * const destination = new gcp.datastream.ConnectionProfile("destination", { * displayName: "BigQuery Destination", * location: "us-central1", * connectionProfileId: "destination-profile", * bigqueryProfile: {}, * }); * const stream5 = new gcp.datastream.Stream("stream5", { * displayName: "Oracle to BigQuery", * location: "us-central1", * streamId: "my-stream", * desiredState: "RUNNING", * sourceConfig: { * sourceConnectionProfile: source.id, * oracleSourceConfig: { * maxConcurrentCdcTasks: 8, * maxConcurrentBackfillTasks: 12, * includeObjects: { * oracleSchemas: [{ * schema: "schema", * oracleTables: [{ * table: "table", * oracleColumns: [{ * column: "column", * }], * }], * }], * }, * excludeObjects: { * oracleSchemas: [{ * schema: "schema", * oracleTables: [{ * table: "table", * oracleColumns: [{ * column: "column", * }], * }], * }], * }, * dropLargeObjects: {}, * }, * }, * destinationConfig: { * destinationConnectionProfile: destination.id, * bigqueryDestinationConfig: { * dataFreshness: "900s", * sourceHierarchyDatasets: { * datasetTemplate: { * location: "us-central1", * }, * }, * }, * }, * backfillAll: { * oracleExcludedObjects: { * oracleSchemas: [{ * schema: "schema", * oracleTables: [{ * table: "table", * oracleColumns: [{ * column: "column", * }], * }], * }], * }, * }, * }); * ``` * ### Datastream Stream Sql Server * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const instance = new gcp.sql.DatabaseInstance("instance", { * name: "sql-server", * databaseVersion: "SQLSERVER_2022_STANDARD", * region: "us-central1", * rootPassword: "root-password", * deletionProtection: true, * settings: { * tier: "db-custom-2-4096", * ipConfiguration: { * authorizedNetworks: [ * { * value: "34.71.242.81", * }, * { * value: "34.72.28.29", * }, * { * value: "34.67.6.157", * }, * { * value: "34.67.234.134", * }, * { * value: "34.72.239.218", * }, * ], * }, * }, * }); * const user = new gcp.sql.User("user", { * name: "user", * instance: instance.name, * password: "password", * }); * const db = new gcp.sql.Database("db", { * name: "db", * instance: instance.name, * }, { * dependsOn: [user], * }); * const source = new gcp.datastream.ConnectionProfile("source", { * displayName: "SQL Server Source", * location: "us-central1", * connectionProfileId: "source-profile", * sqlServerProfile: { * hostname: instance.publicIpAddress, * port: 1433, * username: user.name, * password: user.password, * database: db.name, * }, * }); * const destination = new gcp.datastream.ConnectionProfile("destination", { * displayName: "BigQuery Destination", * location: "us-central1", * connectionProfileId: "destination-profile", * bigqueryProfile: {}, * }); * const _default = new gcp.datastream.Stream("default", { * displayName: "SQL Server to BigQuery", * location: "us-central1", * streamId: "stream", * sourceConfig: { * sourceConnectionProfile: source.id, * sqlServerSourceConfig: { * includeObjects: { * schemas: [{ * schema: "schema", * tables: [{ * table: "table", * }], * }], * }, * transactionLogs: {}, * }, * }, * destinationConfig: { * destinationConnectionProfile: destination.id, * bigqueryDestinationConfig: { * dataFreshness: "900s", * sourceHierarchyDatasets: { * datasetTemplate: { * location: "us-central1", * }, * }, * }, * }, * backfillNone: {}, * }); * ``` * ### Datastream Stream Sql Server Change Tables * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const instance = new gcp.sql.DatabaseInstance("instance", { * name: "sql-server", * databaseVersion: "SQLSERVER_2022_STANDARD", * region: "us-central1", * rootPassword: "root-password", * deletionProtection: true, * settings: { * tier: "db-custom-2-4096", * ipConfiguration: { * authorizedNetworks: [ * { * value: "34.71.242.81", * }, * { * value: "34.72.28.29", * }, * { * value: "34.67.6.157", * }, * { * value: "34.67.234.134", * }, * { * value: "34.72.239.218", * }, * ], * }, * }, * }); * const user = new gcp.sql.User("user", { * name: "user", * instance: instance.name, * password: "password", * }); * const db = new gcp.sql.Database("db", { * name: "db", * instance: instance.name, * }, { * dependsOn: [user], * }); * const source = new gcp.datastream.ConnectionProfile("source", { * displayName: "SQL Server Source", * location: "us-central1", * connectionProfileId: "source-profile", * sqlServerProfile: { * hostname: instance.publicIpAddress, * port: 1433, * username: user.name, * password: user.password, * database: db.name, * }, * }); * const destination = new gcp.datastream.ConnectionProfile("destination", { * displayName: "BigQuery Destination", * location: "us-central1", * connectionProfileId: "destination-profile", * bigqueryProfile: {}, * }); * const _default = new gcp.datastream.Stream("default", { * displayName: "SQL Server to BigQuery", * location: "us-central1", * streamId: "stream", * sourceConfig: { * sourceConnectionProfile: source.id, * sqlServerSourceConfig: { * includeObjects: { * schemas: [{ * schema: "schema", * tables: [{ * table: "table", * }], * }], * }, * changeTables: {}, * }, * }, * destinationConfig: { * destinationConnectionProfile: destination.id, * bigqueryDestinationConfig: { * dataFreshness: "900s", * sourceHierarchyDatasets: { * datasetTemplate: { * location: "us-central1", * }, * }, * }, * }, * backfillNone: {}, * }); * ``` * ### Datastream Stream Mysql Gtid * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const instance = new gcp.sql.DatabaseInstance("instance", { * name: "<%= ctx[:vars]['mysql_name'] %>", * databaseVersion: "MYSQL_8_0", * region: "us-central1", * rootPassword: "<%= ctx[:vars]['mysql_root_password'] %>", * deletionProtection: "<%= ctx[:vars]['deletion_protection'] %>" === "true", * settings: { * tier: "db-custom-2-4096", * ipConfiguration: { * authorizedNetworks: [ * { * value: "34.71.242.81", * }, * { * value: "34.72.28.29", * }, * { * value: "34.67.6.157", * }, * { * value: "34.67.234.134", * }, * { * value: "34.72.239.218", * }, * ], * }, * }, * }); * const user = new gcp.sql.User("user", { * name: "<%= ctx[:vars]['database_user'] %>", * instance: instance.name, * password: "<%= ctx[:vars]['database_password'] %>", * }); * const db = new gcp.sql.Database("db", { * name: "<%= ctx[:vars]['database_name'] %>", * instance: instance.name, * }, { * dependsOn: [user], * }); * const source = new gcp.datastream.ConnectionProfile("source", { * displayName: "MySQL Source", * location: "us-central1", * connectionProfileId: "<%= ctx[:vars]['source_connection_profile_id'] %>", * mysqlProfile: { * hostname: instance.publicIpAddress, * port: 1433, * username: user.name, * password: user.password, * database: db.name, * }, * }); * const destination = new gcp.datastream.ConnectionProfile("destination", { * displayName: "BigQuery Destination", * location: "us-central1", * connectionProfileId: "<%= ctx[:vars]['destination_connection_profile_id'] %>", * bigqueryProfile: {}, * }); * const _default = new gcp.datastream.Stream("default", { * displayName: "MySQL to BigQuery", * location: "us-central1", * streamId: "<%= ctx[:vars]['stream_id'] %>", * sourceConfig: { * sourceConnectionProfile: source.id, * mysqlSourceConfig: { * includeObjects: { * schemas: [{ * schema: "schema", * tables: [{ * table: "table", * }], * }], * }, * gtid: {}, * }, * }, * destinationConfig: { * destinationConnectionProfile: destination.id, * bigqueryDestinationConfig: { * dataFreshness: "900s", * sourceHierarchyDatasets: { * datasetTemplate: { * location: "us-central1", * }, * }, * }, * }, * backfillNone: {}, * }); * ``` * ### Datastream Stream Postgresql Bigquery Dataset Id * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * import * as random from "@pulumi/random"; * * const postgres = new gcp.bigquery.Dataset("postgres", { * datasetId: "postgres", * friendlyName: "postgres", * description: "Database of postgres", * location: "us-central1", * }); * const destinationConnectionProfile2 = new gcp.datastream.ConnectionProfile("destination_connection_profile2", { * displayName: "Connection profile", * location: "us-central1", * connectionProfileId: "dest-profile", * bigqueryProfile: {}, * }); * const instance = new gcp.sql.DatabaseInstance("instance", { * name: "instance-name", * databaseVersion: "MYSQL_8_0", * region: "us-central1", * settings: { * tier: "db-f1-micro", * backupConfiguration: { * enabled: true, * binaryLogEnabled: true, * }, * ipConfiguration: { * authorizedNetworks: [ * { * value: "34.71.242.81", * }, * { * value: "34.72.28.29", * }, * { * value: "34.67.6.157", * }, * { * value: "34.67.234.134", * }, * { * value: "34.72.239.218", * }, * ], * }, * }, * deletionProtection: false, * }); * const pwd = new random.index.Password("pwd", { * length: 16, * special: false, * }); * const user = new gcp.sql.User("user", { * name: "my-user", * instance: instance.name, * host: "%", * password: pwd.result, * }); * const sourceConnectionProfile = new gcp.datastream.ConnectionProfile("source_connection_profile", { * displayName: "Source connection profile", * location: "us-central1", * connectionProfileId: "source-profile", * mysqlProfile: { * hostname: instance.publicIpAddress, * username: user.name, * password: user.password, * }, * }); * const _default = new gcp.datastream.Stream("default", { * displayName: "postgres to bigQuery", * location: "us-central1", * streamId: "postgres-bigquery", * sourceConfig: { * sourceConnectionProfile: sourceConnectionProfile.id, * mysqlSourceConfig: {}, * }, * destinationConfig: { * destinationConnectionProfile: destinationConnectionProfile2.id, * bigqueryDestinationConfig: { * dataFreshness: "900s", * singleTargetDataset: { * datasetId: postgres.id, * }, * }, * }, * backfillAll: {}, * }); * const db = new gcp.sql.Database("db", { * instance: instance.name, * name: "db", * }); * ``` * ### Datastream Stream Bigquery * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * import * as random from "@pulumi/random"; * * const project = gcp.organizations.getProject({}); * const instance = new gcp.sql.DatabaseInstance("instance", { * name: "my-instance", * databaseVersion: "MYSQL_8_0", * region: "us-central1", * settings: { * tier: "db-f1-micro", * backupConfiguration: { * enabled: true, * binaryLogEnabled: true, * }, * ipConfiguration: { * authorizedNetworks: [ * { * value: "34.71.242.81", * }, * { * value: "34.72.28.29", * }, * { * value: "34.67.6.157", * }, * { * value: "34.67.234.134", * }, * { * value: "34.72.239.218", * }, * ], * }, * }, * deletionProtection: true, * }); * const db = new gcp.sql.Database("db", { * instance: instance.name, * name: "db", * }); * const pwd = new random.index.Password("pwd", { * length: 16, * special: false, * }); * const user = new gcp.sql.User("user", { * name: "user", * instance: instance.name, * host: "%", * password: pwd.result, * }); * const sourceConnectionProfile = new gcp.datastream.ConnectionProfile("source_connection_profile", { * displayName: "Source connection profile", * location: "us-central1", * connectionProfileId: "source-profile", * mysqlProfile: { * hostname: instance.publicIpAddress, * username: user.name, * password: user.password, * }, * }); * const bqSa = gcp.bigquery.getDefaultServiceAccount({}); * const bigqueryKeyUser = new gcp.kms.CryptoKeyIAMMember("bigquery_key_user", { * cryptoKeyId: "bigquery-kms-name", * role: "roles/cloudkms.cryptoKeyEncrypterDecrypter", * member: bqSa.then(bqSa => `serviceAccount:${bqSa.email}`), * }); * const destinationConnectionProfile = new gcp.datastream.ConnectionProfile("destination_connection_profile", { * displayName: "Connection profile", * location: "us-central1", * connectionProfileId: "destination-profile", * bigqueryProfile: {}, * }); * const _default = new gcp.datastream.Stream("default", { * streamId: "my-stream", * location: "us-central1", * displayName: "my stream", * sourceConfig: { * sourceConnectionProfile: sourceConnectionProfile.id, * mysqlSourceConfig: {}, * }, * destinationConfig: { * destinationConnectionProfile: destinationConnectionProfile.id, * bigqueryDestinationConfig: { * sourceHierarchyDatasets: { * datasetTemplate: { * location: "us-central1", * kmsKeyName: "bigquery-kms-name", * }, * }, * }, * }, * backfillNone: {}, * }, { * dependsOn: [bigqueryKeyUser], * }); * ``` * ### Datastream Stream Bigquery Cross Project Source Hierachy * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * import * as random from "@pulumi/random"; * import * as time from "@pulumiverse/time"; * * const project = gcp.organizations.getProject({}); * const cross_project_dataset = new gcp.organizations.Project("cross-project-dataset", { * projectId: "tf-test_11171", * name: "tf-test_40472", * orgId: "123456789", * billingAccount: "000000-0000000-0000000-000000", * deletionPolicy: "DELETE", * }); * const wait60Seconds = new time.Sleep("wait_60_seconds", {createDuration: "60s"}, { * dependsOn: [cross_project_dataset], * }); * const bigquery = new gcp.projects.Service("bigquery", { * project: cross_project_dataset.projectId, * service: "bigquery.googleapis.com", * disableOnDestroy: false, * }, { * dependsOn: [wait60Seconds], * }); * const datastreamBigqueryAdmin = new gcp.projects.IAMMember("datastream_bigquery_admin", { * project: cross_project_dataset.projectId, * role: "roles/bigquery.admin", * member: project.then(project => `serviceAccount:service-${project.number}@gcp-sa-datastream.iam.gserviceaccount.com`), * }, { * dependsOn: [wait60Seconds], * }); * const instance = new gcp.sql.DatabaseInstance("instance", { * name: "my-instance", * databaseVersion: "MYSQL_8_0", * region: "us-central1", * settings: { * tier: "db-f1-micro", * backupConfiguration: { * enabled: true, * binaryLogEnabled: true, * }, * ipConfiguration: { * authorizedNetworks: [ * { * value: "34.71.242.81", * }, * { * value: "34.72.28.29", * }, * { * value: "34.67.6.157", * }, * { * value: "34.67.234.134", * }, * { * value: "34.72.239.218", * }, * ], * }, * }, * deletionProtection: true, * }); * const db = new gcp.sql.Database("db", { * instance: instance.name, * name: "db", * }); * const pwd = new random.index.Password("pwd", { * length: 16, * special: false, * }); * const user = new gcp.sql.User("user", { * name: "user", * instance: instance.name, * host: "%", * password: pwd.result, * }); * const sourceConnectionProfile = new gcp.datastream.ConnectionProfile("source_connection_profile", { * displayName: "Source connection profile", * location: "us-central1", * connectionProfileId: "source-profile", * mysqlProfile: { * hostname: instance.publicIpAddress, * username: user.name, * password: user.password, * }, * }); * const destinationConnectionProfile = new gcp.datastream.ConnectionProfile("destination_connection_profile", { * displayName: "Connection profile", * location: "us-central1", * connectionProfileId: "destination-profile", * bigqueryProfile: {}, * }); * const _default = new gcp.datastream.Stream("default", { * streamId: "my-stream", * location: "us-central1", * displayName: "my stream", * sourceConfig: { * sourceConnectionProfile: sourceConnectionProfile.id, * mysqlSourceConfig: {}, * }, * destinationConfig: { * destinationConnectionProfile: destinationConnectionProfile.id, * bigqueryDestinationConfig: { * sourceHierarchyDatasets: { * datasetTemplate: { * location: "us-central1", * }, * projectId: cross_project_dataset.projectId, * }, * }, * }, * backfillNone: {}, * }); * ``` * ### Datastream Stream Bigquery Append Only * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * import * as random from "@pulumi/random"; * * const project = gcp.organizations.getProject({}); * const instance = new gcp.sql.DatabaseInstance("instance", { * name: "my-instance", * databaseVersion: "MYSQL_8_0", * region: "us-central1", * settings: { * tier: "db-f1-micro", * backupConfiguration: { * enabled: true, * binaryLogEnabled: true, * }, * ipConfiguration: { * authorizedNetworks: [ * { * value: "34.71.242.81", * }, * { * value: "34.72.28.29", * }, * { * value: "34.67.6.157", * }, * { * value: "34.67.234.134", * }, * { * value: "34.72.239.218", * }, * ], * }, * }, * deletionProtection: true, * }); * const db = new gcp.sql.Database("db", { * instance: instance.name, * name: "db", * }); * const pwd = new random.index.Password("pwd", { * length: 16, * special: false, * }); * const user = new gcp.sql.User("user", { * name: "user", * instance: instance.name, * host: "%", * password: pwd.result, * }); * const sourceConnectionProfile = new gcp.datastream.ConnectionProfile("source_connection_profile", { * displayName: "Source connection profile", * location: "us-central1", * connectionProfileId: "source-profile", * mysqlProfile: { * hostname: instance.publicIpAddress, * username: user.name, * password: user.password, * }, * }); * const destinationConnectionProfile = new gcp.datastream.ConnectionProfile("destination_connection_profile", { * displayName: "Connection profile", * location: "us-central1", * connectionProfileId: "destination-profile", * bigqueryProfile: {}, * }); * const _default = new gcp.datastream.Stream("default", { * streamId: "my-stream", * location: "us-central1", * displayName: "my stream", * sourceConfig: { * sourceConnectionProfile: sourceConnectionProfile.id, * mysqlSourceConfig: {}, * }, * destinationConfig: { * destinationConnectionProfile: destinationConnectionProfile.id, * bigqueryDestinationConfig: { * sourceHierarchyDatasets: { * datasetTemplate: { * location: "us-central1", * }, * }, * appendOnly: {}, * }, * }, * backfillNone: {}, * }); * ``` * ### Datastream Stream Bigquery Blmt * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * import * as random from "@pulumi/random"; * * const project = gcp.organizations.getProject({}); * const instance = new gcp.sql.DatabaseInstance("instance", { * name: "blmt-instance", * databaseVersion: "MYSQL_8_0", * region: "us-central1", * settings: { * tier: "db-f1-micro", * ipConfiguration: { * authorizedNetworks: [ * { * value: "34.71.242.81", * }, * { * value: "34.72.28.29", * }, * { * value: "34.67.6.157", * }, * { * value: "34.67.234.134", * }, * { * value: "34.72.239.218", * }, * ], * }, * }, * deletionProtection: true, * }); * const db = new gcp.sql.Database("db", { * instance: instance.name, * name: "db", * }); * const pwd = new random.index.Password("pwd", { * length: 16, * special: false, * }); * const user = new gcp.sql.User("user", { * name: "user", * instance: instance.name, * host: "%", * password: pwd.result, * }); * const blmtBucket = new gcp.storage.Bucket("blmt_bucket", { * name: "blmt-bucket", * location: "us-central1", * forceDestroy: true, * }); * const blmtConnection = new gcp.bigquery.Connection("blmt_connection", { * project: project.then(project => project.projectId), * location: "us-central1", * connectionId: "blmt-connection", * friendlyName: "Datastream BLMT Test Connection", * description: "Connection for Datastream BLMT test", * cloudResource: {}, * }); * const blmtConnectionBucketAdmin = new gcp.storage.BucketIAMMember("blmt_connection_bucket_admin", { * bucket: blmtBucket.name, * role: "roles/storage.admin", * member: blmtConnection.cloudResource.apply(cloudResource => `serviceAccount:${cloudResource?.serviceAccountId}`), * }); * const sourceConnectionProfile = new gcp.datastream.ConnectionProfile("source_connection_profile", { * displayName: "Source connection profile", * location: "us-central1", * connectionProfileId: "blmt-source-profile", * mysqlProfile: { * hostname: instance.publicIpAddress, * username: user.name, * password: user.password, * }, * }); * const destinationConnectionProfile = new gcp.datastream.ConnectionProfile("destination_connection_profile", { * displayName: "Connection profile", * location: "us-central1", * connectionProfileId: "blmt-destination-profile", * bigqueryProfile: {}, * }); * const _default = new gcp.datastream.Stream("default", { * streamId: "blmt-stream", * location: "us-central1", * displayName: "My BLMT stream", * sourceConfig: { * sourceConnectionProfile: sourceConnectionProfile.id, * mysqlSourceConfig: {}, * }, * destinationConfig: { * destinationConnectionProfile: destinationConnectionProfile.id, * bigqueryDestinationConfig: { * sourceHierarchyDatasets: { * datasetTemplate: { * location: "us-central1", * }, * }, * blmtConfig: { * bucket: blmtBucket.name, * connectionName: pulumi.all([blmtConnection.project, blmtConnection.location, blmtConnection.connectionId]).apply(([project, location, connectionId]) => `${project}.${location}.${connectionId}`), * fileFormat: "PARQUET", * tableFormat: "ICEBERG", * rootPath: "/", * }, * appendOnly: {}, * }, * }, * backfillNone: {}, * }); * ``` * ### Datastream Stream Rule Sets Bigquery * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const project = gcp.organizations.getProject({}); * const stream = new gcp.datastream.Stream("stream", { * streamId: "rules-stream", * location: "us-central1", * displayName: "BigQuery Stream with Rules", * sourceConfig: { * sourceConnectionProfile: "rules-source-profile", * mysqlSourceConfig: { * includeObjects: { * mysqlDatabases: [{ * database: "my_database", * }], * }, * binaryLogPosition: {}, * }, * }, * destinationConfig: { * destinationConnectionProfile: "rules-dest-profile", * bigqueryDestinationConfig: { * singleTargetDataset: { * datasetId: "rules-project:rules-dataset", * }, * }, * }, * backfillNone: {}, * ruleSets: [ * { * objectFilter: { * sourceObjectIdentifier: { * mysqlIdentifier: { * database: "test_database", * table: "test_table_1", * }, * }, * }, * customizationRules: [ * { * bigqueryClustering: { * columns: ["user_id"], * }, * }, * { * bigqueryPartitioning: { * ingestionTimePartition: {}, * }, * }, * ], * }, * { * objectFilter: { * sourceObjectIdentifier: { * mysqlIdentifier: { * database: "test_database", * table: "test_table_2", * }, * }, * }, * customizationRules: [ * { * bigqueryClustering: { * columns: ["event_time"], * }, * }, * { * bigqueryPartitioning: { * timeUnitPartition: { * column: "event_time", * partitioningTimeGranularity: "PARTITIONING_TIME_GRANULARITY_DAY", * }, * }, * }, * ], * }, * ], * }); * ``` * ### Datastream Stream Mongodb * * ```typescript * import * as pulumi from "@pulumi/pulumi"; * import * as gcp from "@pulumi/gcp"; * * const _default = new gcp.datastream.Stream("default", { * displayName: "Mongodb to BigQuery", * location: "us-central1", * streamId: "mongodb-stream", * sourceConfig: { * sourceConnectionProfile: "source-profile", * mongodbSourceConfig: { * includeObjects: { * databases: [{ * database: "mydb", * collections: [ * { * collection: "mycollection1", * }, * { * collection: "mycollection2", * }, * ], * }], * }, * excludeeObjects: [{ * databases: [{ * database: "mydb", * collections: [{ * fields: [{ * field: "excludedField", * }], * }], * }], * }], * }, * }, * destinationConfig: { * destinationConnectionProfile: "destination-profile", * bigqueryDestinationConfig: { * dataFreshness: "900s", * sourceHierarchyDatasets: { * datasetTemplate: { * location: "us-central1", * }, * }, * }, * }, * backfillNone: {}, * }); * ``` * * ## Import * * Stream can be imported using any of these accepted formats: * * * `projects/{{project}}/locations/{{location}}/streams/{{stream_id}}` * * `{{project}}/{{location}}/{{stream_id}}` * * `{{location}}/{{stream_id}}` * * When using the `pulumi import` command, Stream can be imported using one of the formats above. For example: * * ```sh * $ pulumi import gcp:datastream/stream:Stream default projects/{{project}}/locations/{{location}}/streams/{{stream_id}} * $ pulumi import gcp:datastream/stream:Stream default {{project}}/{{location}}/{{stream_id}} * $ pulumi import gcp:datastream/stream:Stream default {{location}}/{{stream_id}} * ``` */ export declare class Stream extends pulumi.CustomResource { /** * Get an existing Stream resource's state with the given name, ID, and optional extra * properties used to qualify the lookup. * * @param name The _unique_ name of the resulting resource. * @param id The _unique_ provider ID of the resource to lookup. * @param state Any extra arguments used during the lookup. * @param opts Optional settings to control the behavior of the CustomResource. */ static get(name: string, id: pulumi.Input, state?: StreamState, opts?: pulumi.CustomResourceOptions): Stream; /** * Returns true if the given object is an instance of Stream. This is designed to work even * when multiple copies of the Pulumi SDK have been loaded into the same process. */ static isInstance(obj: any): obj is Stream; /** * Backfill strategy to automatically backfill the Stream's objects. Specific objects can be excluded. * Structure is documented below. */ readonly backfillAll: pulumi.Output; /** * Backfill strategy to disable automatic backfill for the Stream's objects. */ readonly backfillNone: pulumi.Output; /** * Create the stream without validating it. */ readonly createWithoutValidation: pulumi.Output; /** * A reference to a KMS encryption key. If provided, it will be used to encrypt the data. If left blank, data * will be encrypted using an internal Stream-specific encryption key provisioned through KMS. */ readonly customerManagedEncryptionKey: pulumi.Output; /** * Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. * When a 'terraform destroy' or 'pulumi up' would delete the resource, * the command will fail if this field is set to "PREVENT" in Terraform state. * When set to "ABANDON", the command will remove the resource from Terraform * management without updating or deleting the resource in the API. * When set to "DELETE", deleting the resource is allowed. */ readonly deletionPolicy: pulumi.Output; /** * Desired state of the Stream. Set this field to `RUNNING` to start the stream, * `NOT_STARTED` to create the stream without starting and `PAUSED` to pause * the stream from a `RUNNING` state. * Possible values: NOT_STARTED, RUNNING, PAUSED. Default: NOT_STARTED */ readonly desiredState: pulumi.Output; /** * Destination connection profile configuration. * Structure is documented below. */ readonly destinationConfig: pulumi.Output; /** * Display name. */ readonly displayName: pulumi.Output; /** * All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services. */ readonly effectiveLabels: pulumi.Output<{ [key: string]: string; }>; /** * Labels. * **Note**: This field is non-authoritative, and will only manage the labels present in your configuration. * Please refer to the field `effectiveLabels` for all of the labels present on the resource. */ readonly labels: pulumi.Output<{ [key: string]: string; } | undefined>; /** * The name of the location this stream is located in. */ readonly location: pulumi.Output; /** * The stream's name. */ readonly name: pulumi.Output; /** * The ID of the project in which the resource belongs. * If it is not provided, the provider project is used. */ readonly project: pulumi.Output; /** * The combination of labels configured directly on the resource * and default labels configured on the provider. */ readonly pulumiLabels: pulumi.Output<{ [key: string]: string; }>; /** * Rule sets to apply to the stream. * Structure is documented below. */ readonly ruleSets: pulumi.Output; /** * Source connection profile configuration. * Structure is documented below. */ readonly sourceConfig: pulumi.Output; /** * The state of the stream. */ readonly state: pulumi.Output; /** * The stream identifier. */ readonly streamId: pulumi.Output; /** * Create a Stream resource with the given unique name, arguments, and options. * * @param name The _unique_ name of the resource. * @param args The arguments to use to populate this resource's properties. * @param opts A bag of options that control this resource's behavior. */ constructor(name: string, args: StreamArgs, opts?: pulumi.CustomResourceOptions); } /** * Input properties used for looking up and filtering Stream resources. */ export interface StreamState { /** * Backfill strategy to automatically backfill the Stream's objects. Specific objects can be excluded. * Structure is documented below. */ backfillAll?: pulumi.Input; /** * Backfill strategy to disable automatic backfill for the Stream's objects. */ backfillNone?: pulumi.Input; /** * Create the stream without validating it. */ createWithoutValidation?: pulumi.Input; /** * A reference to a KMS encryption key. If provided, it will be used to encrypt the data. If left blank, data * will be encrypted using an internal Stream-specific encryption key provisioned through KMS. */ customerManagedEncryptionKey?: pulumi.Input; /** * Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. * When a 'terraform destroy' or 'pulumi up' would delete the resource, * the command will fail if this field is set to "PREVENT" in Terraform state. * When set to "ABANDON", the command will remove the resource from Terraform * management without updating or deleting the resource in the API. * When set to "DELETE", deleting the resource is allowed. */ deletionPolicy?: pulumi.Input; /** * Desired state of the Stream. Set this field to `RUNNING` to start the stream, * `NOT_STARTED` to create the stream without starting and `PAUSED` to pause * the stream from a `RUNNING` state. * Possible values: NOT_STARTED, RUNNING, PAUSED. Default: NOT_STARTED */ desiredState?: pulumi.Input; /** * Destination connection profile configuration. * Structure is documented below. */ destinationConfig?: pulumi.Input; /** * Display name. */ displayName?: pulumi.Input; /** * All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Pulumi, other clients and services. */ effectiveLabels?: pulumi.Input<{ [key: string]: pulumi.Input; } | undefined>; /** * Labels. * **Note**: This field is non-authoritative, and will only manage the labels present in your configuration. * Please refer to the field `effectiveLabels` for all of the labels present on the resource. */ labels?: pulumi.Input<{ [key: string]: pulumi.Input; } | undefined>; /** * The name of the location this stream is located in. */ location?: pulumi.Input; /** * The stream's name. */ name?: pulumi.Input; /** * The ID of the project in which the resource belongs. * If it is not provided, the provider project is used. */ project?: pulumi.Input; /** * The combination of labels configured directly on the resource * and default labels configured on the provider. */ pulumiLabels?: pulumi.Input<{ [key: string]: pulumi.Input; } | undefined>; /** * Rule sets to apply to the stream. * Structure is documented below. */ ruleSets?: pulumi.Input[] | undefined>; /** * Source connection profile configuration. * Structure is documented below. */ sourceConfig?: pulumi.Input; /** * The state of the stream. */ state?: pulumi.Input; /** * The stream identifier. */ streamId?: pulumi.Input; } /** * The set of arguments for constructing a Stream resource. */ export interface StreamArgs { /** * Backfill strategy to automatically backfill the Stream's objects. Specific objects can be excluded. * Structure is documented below. */ backfillAll?: pulumi.Input; /** * Backfill strategy to disable automatic backfill for the Stream's objects. */ backfillNone?: pulumi.Input; /** * Create the stream without validating it. */ createWithoutValidation?: pulumi.Input; /** * A reference to a KMS encryption key. If provided, it will be used to encrypt the data. If left blank, data * will be encrypted using an internal Stream-specific encryption key provisioned through KMS. */ customerManagedEncryptionKey?: pulumi.Input; /** * Whether Terraform will be prevented from destroying the resource. Defaults to DELETE. * When a 'terraform destroy' or 'pulumi up' would delete the resource, * the command will fail if this field is set to "PREVENT" in Terraform state. * When set to "ABANDON", the command will remove the resource from Terraform * management without updating or deleting the resource in the API. * When set to "DELETE", deleting the resource is allowed. */ deletionPolicy?: pulumi.Input; /** * Desired state of the Stream. Set this field to `RUNNING` to start the stream, * `NOT_STARTED` to create the stream without starting and `PAUSED` to pause * the stream from a `RUNNING` state. * Possible values: NOT_STARTED, RUNNING, PAUSED. Default: NOT_STARTED */ desiredState?: pulumi.Input; /** * Destination connection profile configuration. * Structure is documented below. */ destinationConfig: pulumi.Input; /** * Display name. */ displayName: pulumi.Input; /** * Labels. * **Note**: This field is non-authoritative, and will only manage the labels present in your configuration. * Please refer to the field `effectiveLabels` for all of the labels present on the resource. */ labels?: pulumi.Input<{ [key: string]: pulumi.Input; } | undefined>; /** * The name of the location this stream is located in. */ location: pulumi.Input; /** * The ID of the project in which the resource belongs. * If it is not provided, the provider project is used. */ project?: pulumi.Input; /** * Rule sets to apply to the stream. * Structure is documented below. */ ruleSets?: pulumi.Input[] | undefined>; /** * Source connection profile configuration. * Structure is documented below. */ sourceConfig: pulumi.Input; /** * The stream identifier. */ streamId: pulumi.Input; } //# sourceMappingURL=stream.d.ts.map