@google-cloud/bigquery/build/src/table.js

"use strict";
/*!
 * Copyright 2014 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
Object.defineProperty(exports, "__esModule", { value: true });
exports.Table = void 0;
const common_1 = require("@google-cloud/common");
const paginator_1 = require("@google-cloud/paginator");
const promisify_1 = require("@google-cloud/promisify");
const arrify = require("arrify");
const Big = require("big.js");
const extend = require("extend");
const events_1 = require("events");
const fs = require("fs");
const is = require("is");
const path = require("path");
const streamEvents = require("stream-events");
const uuid = require("uuid");
const _1 = require(".");
const stream_1 = require("stream");
const rowQueue_1 = require("./rowQueue");
// eslint-disable-next-line @typescript-eslint/no-var-requires
const duplexify = require('duplexify');
/**
 * The file formats accepted by BigQuery.
 *
 * @type {object}
 * @private
 */
const FORMATS = {
    avro: 'AVRO',
    csv: 'CSV',
    export_metadata: 'DATASTORE_BACKUP',
    json: 'NEWLINE_DELIMITED_JSON',
    orc: 'ORC',
    parquet: 'PARQUET',
};
/**
 * Table objects are returned by methods such as
 * {@link Dataset#table}, {@link Dataset#createTable}, and
 * {@link Dataset#getTables}.
 *
 * @class
 * @param {Dataset} dataset {@link Dataset} instance.
 * @param {string} id The ID of the table.
 * @param {object} [options] Table options.
 * @param {string} [options.location] The geographic location of the table, by
 *      default this value is inherited from the dataset. This can be used to
 *      configure the location of all jobs created through a table instance. It
 *      cannot be used to set the actual location of the table. This value will
 *      be superseded by any API responses containing location data for the
 *      table.
 *
 * @example
 * ```
 * const {BigQuery} = require('@google-cloud/bigquery');
 * const bigquery = new BigQuery();
 * const dataset = bigquery.dataset('my-dataset');
 *
 * const table = dataset.table('my-table');
 * ```
 */
class Table extends common_1.ServiceObject {
    createReadStream(options) {
        // placeholder body, overwritten in constructor
        return new paginator_1.ResourceStream({}, () => { });
    }
    constructor(dataset, id, options) {
        const methods = {
            /**
             * @callback CreateTableCallback
             * @param {?Error} err Request error, if any.
             * @param {Table} table The table.
             * @param {object} apiResponse The full API response body.
             */
            /**
             * @typedef {array} CreateTableResponse
             * @property {Table} 0 The table.
             * @property {object} 1 The full API response body.
             */
            /**
             * Create a table.
             *
             * @method Table#create
             * @param {object} [options] See {@link Dataset#createTable}.
             * @param {CreateTableCallback} [callback]
             * @param {?error} callback.err An error returned while making this
             *     request.
             * @param {Table} callback.table The new {@link Table}.
             * @param {object} callback.apiResponse The full API response.
             * @returns {Promise<CreateTableResponse>}
             *
             * @example
             * ```
             * const {BigQuery} = require('@google-cloud/bigquery');
             * const bigquery = new BigQuery();
             * const dataset = bigquery.dataset('my-dataset');
             *
             * const table = dataset.table('my-table');
             *
             * table.create((err, table, apiResponse) => {
             *   if (!err) {
             *     // The table was created successfully.
             *   }
             * });
             *
             * //-
             * // If the callback is omitted, we'll return a Promise.
             * //-
             * table.create().then((data) => {
             *   const table = data[0];
             *   const apiResponse = data[1];
             * });
             * ```
             */
            create: true,
            /**
             * @callback DeleteTableCallback
             * @param {?Error} err Request error, if any.
             * @param {object} apiResponse The full API response.
             */
            /**
             * @typedef {array} DeleteTableResponse
             * @property {object} 0 The full API response.
             */
            /**
             * Delete a table and all its data.
             *
             * See {@link https://cloud.google.com/bigquery/docs/reference/v2/tables/delete| Tables: delete API Documentation}
             *
             * @method Table#delete
             * @param {DeleteTableCallback} [callback]
             * @param {?error} callback.err An error returned while making this
             *     request.
             * @param {object} callback.apiResponse The full API response.
             * @returns {Promise<DeleteTableResponse>}
             *
             * @example
             * ```
             * const {BigQuery} = require('@google-cloud/bigquery');
             * const bigquery = new BigQuery();
             * const dataset = bigquery.dataset('my-dataset');
             *
             * const table = dataset.table('my-table');
             *
             * table.delete((err, apiResponse) => {});
             *
             * //-
             * // If the callback is omitted, we'll return a Promise.
             * //-
             * table.delete().then((data) => {
             *   const apiResponse = data[0];
             * });
             * ```
             */
            delete: true,
            /**
             * @callback TableExistsCallback
             * @param {?Error} err Request error, if any.
             * @param {boolean} exists Indicates if the table exists.
             */
            /**
             * @typedef {array} TableExistsCallback
             * @property {boolean} 0 Indicates if the table exists.
             */
            /**
             * Check if the table exists.
             *
             * @method Table#exists
             * @param {TableExistsCallback} [callback]
             * @param {?error} callback.err An error returned while making this
             *     request.
             * @param {boolean} callback.exists Whether the table exists or not.
             * @returns {Promise<TableExistsCallback>}
             *
             * @example
             * ```
             * const {BigQuery} = require('@google-cloud/bigquery');
             * const bigquery = new BigQuery();
             * const dataset = bigquery.dataset('my-dataset');
             *
             * const table = dataset.table('my-table');
             *
             * table.exists((err, exists) => {});
             *
             * //-
             * // If the callback is omitted, we'll return a Promise.
             * //-
             * table.exists().then((data) => {
             *   const exists = data[0];
             * });
             * ```
             */
            exists: true,
            /**
             * @callback GetTableCallback
             * @param {?Error} err Request error, if any.
             * @param {Table} table The table.
             * @param {object} apiResponse The full API response body.
             */
            /**
             * @typedef {array} GetTableResponse
             * @property {Table} 0 The table.
             * @property {object} 1 The full API response body.
             */
            /**
             * Get a table if it exists.
             *
             * You may optionally use this to "get or create" an object by providing
             * an object with `autoCreate` set to `true`. Any extra configuration that
             * is normally required for the `create` method must be contained within
             * this object as well.
             *
             * If you wish to get a selection of metadata instead of the full table metadata
             * (retrieved by both Table#get by default and by Table#getMetadata), use
             * the `options` parameter to set the `view` and/or `selectedFields` query parameters.
             *
             * See {@link https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/get#TableMetadataView| Tables.get and TableMetadataView }
             *
             * @method Table#get
             * @param {options} [options] Configuration object.
             * @param {boolean} [options.autoCreate=false] Automatically create the
             *     object if it does not exist.
             * @param {function} [callback]
             * @param {?error} callback.err An error returned while making this
             *     request.
             * @param {Table} callback.table The {@link Table}.
             * @param {object} callback.apiResponse The full API response.
             * @returns {Promise<GetTableResponse>}
             *
             * @example
             * ```
             * const {BigQuery} = require('@google-cloud/bigquery');
             * const bigquery = new BigQuery();
             * const dataset = bigquery.dataset('my-dataset');
             *
             * const table = dataset.table('my-table');
             *
             * const options = {
             *   view: "BASIC"
             * }
             *
             * table.get((err, table, apiResponse) => {
             *   // `table.metadata` has been populated.
             * });
             *
             * table.get(options, (err, table, apiResponse) => {
             *   // A selection of `table.metadata` has been populated
             * })
             *
             * //-
             * // If the callback is omitted, we'll return a Promise.
             * //-
             * table.get().then((data) => {
             *   const table = data[0];
             *   const apiResponse = data[1];
             * });
             * ```
             */
            get: true,
            /**
             * @callback GetTableMetadataCallback
             * @param {?Error} err Request error, if any.
             * @param {object} metadata The table metadata.
             * @param {object} apiResponse The full API response.
             */
            /**
             * @typedef {array} GetTableMetadataResponse
             * @property {object} 0 The table metadata.
             * @property {object} 1 The full API response.
             */
            /**
             * Return the metadata associated with the Table.
             *
             * See {@link https://cloud.google.com/bigquery/docs/reference/v2/tables/get| Tables: get API Documentation}
             *
             * @method Table#getMetadata
             * @param {GetTableMetadataCallback} [callback] The callback function.
             * @param {?error} callback.err An error returned while making this
             *     request.
             * @param {object} callback.metadata The metadata of the Table.
             * @param {object} callback.apiResponse The full API response.
             * @returns {Promise<GetTableMetadataResponse>}
             *
             * @example
             * ```
             * const {BigQuery} = require('@google-cloud/bigquery');
             * const bigquery = new BigQuery();
             * const dataset = bigquery.dataset('my-dataset');
             *
             * const table = dataset.table('my-table');
             *
             * table.getMetadata((err, metadata, apiResponse) => {});
             *
             * //-
             * // If the callback is omitted, we'll return a Promise.
             * //-
             * table.getMetadata().then((data) => {
             *   const metadata = data[0];
             *   const apiResponse = data[1];
             * });
             * ```
             */
            getMetadata: true,
        };
        super({
            parent: dataset,
            baseUrl: '/tables',
            id,
            createMethod: dataset.createTable.bind(dataset),
            methods,
        });
        if (options && options.location) {
            this.location = options.location;
        }
        this.bigQuery = dataset.bigQuery;
        this.dataset = dataset;
        // Catch all for read-modify-write cycle
        // https://cloud.google.com/bigquery/docs/api-performance#read-patch-write
        this.interceptors.push({
            request: (reqOpts) => {
                if (reqOpts.method === 'PATCH' && reqOpts.json.etag) {
                    reqOpts.headers = reqOpts.headers || {};
                    reqOpts.headers['If-Match'] = reqOpts.json.etag;
                }
                return reqOpts;
            },
        });
        /**
         * Create a readable stream of the rows of data in your table. This method
         * is simply a wrapper around {@link Table#getRows}.
         *
         * See {@link https://cloud.google.com/bigquery/docs/reference/v2/tabledata/list| Tabledata: list API Documentation}
         *
         * @returns {ReadableStream}
         *
         * @example
         * ```
         * const {BigQuery} = require('@google-cloud/bigquery');
         * const bigquery = new BigQuery();
         * const dataset = bigquery.dataset('my-dataset');
         * const table = dataset.table('my-table');
         *
         * table.createReadStream(options)
         *   .on('error', console.error)
         *   .on('data', row => {})
         *   .on('end', function() {
         *     // All rows have been retrieved.
         *   });
         *
         * //-
         * // If you anticipate many results, you can end a stream early to prevent
         * // unnecessary processing and API requests.
         * //-
         * table.createReadStream()
         *   .on('data', function(row) {
         *     this.end();
         *   });
         * ```
         */
        this.createReadStream = paginator_1.paginator.streamify('getRows');
    }
    /**
     * Convert a comma-separated name:type string to a table schema object.
     *
     * @static
     * @private
     *
     * @param {string} str Comma-separated schema string.
     * @returns {object} Table schema in the format the API expects.
     */
    static createSchemaFromString_(str) {
        return str.split(/\s*,\s*/).reduce((acc, pair) => {
            acc.fields.push({
                name: pair.split(':')[0].trim(),
                type: (pair.split(':')[1] || 'STRING').toUpperCase().trim(),
            });
            return acc;
        }, {
            fields: [],
        });
    }
    /**
     * Convert a row entry from native types to their encoded types that the API
     * expects.
     *
     * @static
     * @private
     *
     * @param {*} value The value to be converted.
     * @returns {*} The converted value.
     */
    static encodeValue_(value) {
        var _a;
        if (typeof value === 'undefined' || value === null) {
            return null;
        }
        if (value instanceof Buffer) {
            return value.toString('base64');
        }
        if (value instanceof Big) {
            return value.toFixed();
        }
        const customTypeConstructorNames = [
            'BigQueryDate',
            'BigQueryDatetime',
            'BigQueryInt',
            'BigQueryTime',
            'BigQueryTimestamp',
            'BigQueryRange',
            'Geography',
        ];
        const constructorName = (_a = value.constructor) === null || _a === void 0 ? void 0 : _a.name;
        const isCustomType = customTypeConstructorNames.indexOf(constructorName) > -1;
        if (isCustomType) {
            return value.value;
        }
        if (is.date(value)) {
            return value.toJSON();
        }
        if (is.array(value)) {
            return value.map(Table.encodeValue_);
        }
        if (typeof value === 'object') {
            return Object.keys(value).reduce((acc, key) => {
                acc[key] = Table.encodeValue_(value[key]);
                return acc;
            }, {});
        }
        return value;
    }
    /**
     * @private
     */
    static formatMetadata_(options) {
        const body = extend(true, {}, options);
        if (options.name) {
            body.friendlyName = options.name;
            delete body.name;
        }
        if (is.string(options.schema)) {
            body.schema = Table.createSchemaFromString_(options.schema);
        }
        if (is.array(options.schema)) {
            body.schema = {
                fields: options.schema,
            };
        }
        if (body.schema && body.schema.fields) {
            body.schema.fields = body.schema.fields.map(field => {
                if (field.fields) {
                    field.type = 'RECORD';
                }
                return field;
            });
        }
        if (is.string(options.partitioning)) {
            body.timePartitioning = {
                type: options.partitioning.toUpperCase(),
            };
            delete body.partitioning;
        }
        if (is.string(options.view)) {
            body.view = {
                query: options.view,
                useLegacySql: false,
            };
        }
        return body;
    }
    copy(destination, metadataOrCallback, cb) {
        const metadata = typeof metadataOrCallback === 'object' ? metadataOrCallback : {};
        const callback = typeof metadataOrCallback === 'function' ? metadataOrCallback : cb;
        this.createCopyJob(destination, metadata, (err, job, resp) => {
            if (err) {
                callback(err, resp);
                return;
            }
            job.on('error', callback).on('complete', (metadata) => {
                callback(null, metadata);
            });
        });
    }
    copyFrom(sourceTables, metadataOrCallback, cb) {
        const metadata = typeof metadataOrCallback === 'object' ? metadataOrCallback : {};
        const callback = typeof metadataOrCallback === 'function' ? metadataOrCallback : cb;
        this.createCopyFromJob(sourceTables, metadata, (err, job, resp) => {
            if (err) {
                callback(err, resp);
                return;
            }
            job.on('error', callback).on('complete', metadata => {
                callback(null, metadata);
            });
        });
    }
    createCopyJob(destination, metadataOrCallback, cb) {
        if (!(destination instanceof Table)) {
            throw new Error('Destination must be a Table object.');
        }
        const metadata = typeof metadataOrCallback === 'object'
            ? metadataOrCallback
            : {};
        const callback = typeof metadataOrCallback === 'function' ? metadataOrCallback : cb;
        // eslint-disable-next-line @typescript-eslint/no-explicit-any
        const body = {
            configuration: {
                copy: extend(true, metadata, {
                    destinationTable: {
                        datasetId: destination.dataset.id,
                        projectId: destination.dataset.projectId,
                        tableId: destination.id,
                    },
                    sourceTable: {
                        datasetId: this.dataset.id,
                        projectId: this.dataset.projectId,
                        tableId: this.id,
                    },
                }),
            },
        };
        if (metadata.jobPrefix) {
            body.jobPrefix = metadata.jobPrefix;
            delete metadata.jobPrefix;
        }
        if (this.location) {
            body.location = this.location;
        }
        if (metadata.jobId) {
            body.jobId = metadata.jobId;
            delete metadata.jobId;
        }
        this.bigQuery.createJob(body, callback);
    }
    createCopyFromJob(source, metadataOrCallback, cb) {
        const sourceTables = arrify(source);
        sourceTables.forEach(sourceTable => {
            if (!(sourceTable instanceof Table)) {
                throw new Error('Source must be a Table object.');
            }
        });
        const metadata = typeof metadataOrCallback === 'object' ? metadataOrCallback : {};
        const callback = typeof metadataOrCallback === 'function' ? metadataOrCallback : cb;
        // eslint-disable-next-line @typescript-eslint/no-explicit-any
        const body = {
            configuration: {
                copy: extend(true, metadata, {
                    destinationTable: {
                        datasetId: this.dataset.id,
                        projectId: this.dataset.projectId,
                        tableId: this.id,
                    },
                    sourceTables: sourceTables.map(sourceTable => {
                        return {
                            datasetId: sourceTable.dataset.id,
                            projectId: sourceTable.dataset.projectId,
                            tableId: sourceTable.id,
                        };
                    }),
                }),
            },
        };
        if (metadata.jobPrefix) {
            body.jobPrefix = metadata.jobPrefix;
            delete metadata.jobPrefix;
        }
        if (this.location) {
            body.location = this.location;
        }
        if (metadata.jobId) {
            body.jobId = metadata.jobId;
            delete metadata.jobId;
        }
        this.bigQuery.createJob(body, callback);
    }
    createExtractJob(destination, optionsOrCallback, cb) {
        let options = typeof optionsOrCallback === 'object' ? optionsOrCallback : {};
        const callback = typeof optionsOrCallback === 'function' ? optionsOrCallback : cb;
        options = extend(true, options, {
            destinationUris: arrify(destination).map(dest => {
                if (!common_1.util.isCustomType(dest, 'storage/file')) {
                    throw new Error('Destination must be a File object.');
                }
                // If no explicit format was provided, attempt to find a match from the
                // file's extension. If no match, don't set, and default upstream to
                // CSV.
                const format = path.extname(dest.name).substr(1).toLowerCase();
                if (!options.destinationFormat && !options.format && FORMATS[format]) {
                    options.destinationFormat = FORMATS[format];
                }
                return 'gs://' + dest.bucket.name + '/' + dest.name;
            }),
        });
        if (options.format) {
            options.format = options.format.toLowerCase();
            if (FORMATS[options.format]) {
                options.destinationFormat = FORMATS[options.format];
                delete options.format;
            }
            else {
                throw new Error('Destination format not recognized: ' + options.format);
            }
        }
        if (options.gzip) {
            options.compression = 'GZIP';
            delete options.gzip;
        }
        // eslint-disable-next-line @typescript-eslint/no-explicit-any
        const body = {
            configuration: {
                extract: extend(true, options, {
                    sourceTable: {
                        datasetId: this.dataset.id,
                        projectId: this.dataset.projectId,
                        tableId: this.id,
                    },
                }),
            },
        };
        if (options.jobPrefix) {
            body.jobPrefix = options.jobPrefix;
            delete options.jobPrefix;
        }
        if (this.location) {
            body.location = this.location;
        }
        if (options.jobId) {
            body.jobId = options.jobId;
            delete options.jobId;
        }
        this.bigQuery.createJob(body, callback);
    }
    createLoadJob(source, metadataOrCallback, cb) {
        const metadata = typeof metadataOrCallback === 'object' ? metadataOrCallback : {};
        const callback = typeof metadataOrCallback === 'function' ? metadataOrCallback : cb;
        this._createLoadJob(source, metadata).then(([resp]) => callback(null, resp, resp.metadata), err => callback(err));
    }
    /**
     * @param {string | File | File[]} source
     * @param {JobLoadMetadata} metadata
     * @returns {Promise<JobResponse>}
     * @private
     */
    async _createLoadJob(source, metadata) {
        if (metadata.format) {
            metadata.sourceFormat = FORMATS[metadata.format.toLowerCase()];
            delete metadata.format;
        }
        if (this.location) {
            metadata.location = this.location;
        }
        if (typeof source === 'string') {
            // A path to a file was given. If a sourceFormat wasn't specified, try to
            // find a match from the file's extension.
            const detectedFormat = FORMATS[path.extname(source).substr(1).toLowerCase()];
            if (!metadata.sourceFormat && detectedFormat) {
                metadata.sourceFormat = detectedFormat;
            }
            // Read the file into a new write stream.
            const jobWritable = fs
                .createReadStream(source)
                .pipe(this.createWriteStream_(metadata));
            const [jobResponse] = (await (0, events_1.once)(jobWritable, 'job'));
            return [jobResponse, jobResponse.metadata];
        }
        // eslint-disable-next-line @typescript-eslint/no-explicit-any
        const body = {
            configuration: {
                load: {
                    destinationTable: {
                        projectId: this.dataset.projectId,
                        datasetId: this.dataset.id,
                        tableId: this.id,
                    },
                },
            },
        };
        if (metadata.jobPrefix) {
            body.jobPrefix = metadata.jobPrefix;
            delete metadata.jobPrefix;
        }
        if (metadata.location) {
            body.location = metadata.location;
            delete metadata.location;
        }
        if (metadata.jobId) {
            body.jobId = metadata.jobId;
            delete metadata.jobId;
        }
        extend(true, body.configuration.load, metadata, {
            sourceUris: arrify(source).map(src => {
                if (!common_1.util.isCustomType(src, 'storage/file')) {
                    throw new Error('Source must be a File object.');
                }
                // If no explicit format was provided, attempt to find a match from
                // the file's extension. If no match, don't set, and default upstream
                // to CSV.
                const format = FORMATS[path.extname(src.name).substr(1).toLowerCase()];
                if (!metadata.sourceFormat && format) {
                    body.configuration.load.sourceFormat = format;
                }
                return 'gs://' + src.bucket.name + '/' + src.name;
            }),
        });
        return this.bigQuery.createJob(body);
    }
    createQueryJob(options, callback) {
        return this.dataset.createQueryJob(options, callback);
    }
    /**
     * Run a query scoped to your dataset as a readable object stream.
     *
     * See {@link BigQuery#createQueryStream} for full documentation of this
     * method.
     *
     * @param {object} query See {@link BigQuery#createQueryStream} for full
     *     documentation of this method.
     * @returns {stream} See {@link BigQuery#createQueryStream} for full
     *     documentation of this method.
     */
    createQueryStream(query) {
        return this.dataset.createQueryStream(query);
    }
    /**
     * Creates a write stream. Unlike the public version, this will not
     * automatically poll the underlying job.
     *
     * @private
     *
     * @param {string|object} [metadata] Metadata to set with the load operation.
     *     The metadata object should be in the format of the
     *     {@link https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad| `configuration.load`}
     * property of a Jobs resource. If a string is given, it will be used
     * as the filetype.
     * @param {string} [metadata.jobId] Custom job id.
     * @param {string} [metadata.jobPrefix] Prefix to apply to the job id.
     * @returns {WritableStream}
     */
    createWriteStream_(metadata) {
        metadata = metadata || {};
        if (typeof metadata === 'string') {
            metadata = {
                sourceFormat: FORMATS[metadata.toLowerCase()],
            };
        }
        if (typeof metadata.schema === 'string') {
            metadata.schema = Table.createSchemaFromString_(metadata.schema);
        }
        metadata = extend(true, {
            destinationTable: {
                projectId: this.dataset.projectId,
                datasetId: this.dataset.id,
                tableId: this.id,
            },
        }, metadata);
        let jobId = metadata.jobId || uuid.v4();
        if (metadata.jobId) {
            delete metadata.jobId;
        }
        if (metadata.jobPrefix) {
            jobId = metadata.jobPrefix + jobId;
            delete metadata.jobPrefix;
        }
        const dup = streamEvents(duplexify());
        dup.once('writing', () => {
            common_1.util.makeWritableStream(dup, {
                makeAuthenticatedRequest: this.bigQuery.makeAuthenticatedRequest,
                metadata: {
                    configuration: {
                        load: metadata,
                    },
                    jobReference: {
                        jobId,
                        projectId: this.dataset.projectId,
                        location: this.location,
                    },
                },
                request: {
                    uri: `${this.bigQuery.apiEndpoint}/upload/bigquery/v2/projects/${this.dataset.projectId}/jobs`,
                },
            }, 
            // eslint-disable-next-line @typescript-eslint/no-explicit-any
            (data) => {
                const job = this.bigQuery.job(data.jobReference.jobId, {
                    location: data.jobReference.location,
                    projectId: data.jobReference.projectId,
                });
                job.metadata = data;
                dup.emit('job', job);
            });
        });
        return dup;
    }
    /**
     * Load data into your table from a readable stream of AVRO, CSV, JSON, ORC,
     * or PARQUET data.
     *
     * See {@link https://cloud.google.com/bigquery/docs/reference/v2/jobs/insert| Jobs: insert API Documentation}
     *
     * @param {string|object} [metadata] Metadata to set with the load operation.
     *     The metadata object should be in the format of the
     *     {@link https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad| `configuration.load`}
     * property of a Jobs resource. If a string is given,
     * it will be used as the filetype.
     * @param {string} [metadata.jobId] Custom job id.
     * @param {string} [metadata.jobPrefix] Prefix to apply to the job id.
     * @returns {WritableStream}
     *
     * @throws {Error} If source format isn't recognized.
     *
     * @example
     * ```
     * const {BigQuery} = require('@google-cloud/bigquery');
     * const bigquery = new BigQuery();
     * const dataset = bigquery.dataset('my-dataset');
     * const table = dataset.table('my-table');
     *
     * //-
     * // Load data from a CSV file.
     * //-
     * const request = require('request');
     *
     * const csvUrl = 'http://goo.gl/kSE7z6';
     *
     * const metadata = {
     *   allowJaggedRows: true,
     *   skipLeadingRows: 1
     * };
     *
     * request.get(csvUrl)
     *   .pipe(table.createWriteStream(metadata))
     *   .on('job', (job) => {
     *     // `job` is a Job object that can be used to check the status of the
     *     // request.
     *   })
     *   .on('complete', (job) => {
     *     // The job has completed successfully.
     *   });
     *
     * //-
     * // Load data from a JSON file.
     * //-
     * const fs = require('fs');
     *
     * fs.createReadStream('./test/testdata/testfile.json')
     *   .pipe(table.createWriteStream('json'))
     *   .on('job', (job) => {
     *     // `job` is a Job object that can be used to check the status of the
     *     // request.
     *   })
     *   .on('complete', (job) => {
     *     // The job has completed successfully.
     *   });
     * ```
     */
    createWriteStream(metadata) {
        const stream = this.createWriteStream_(metadata);
        stream.on('prefinish', () => {
            stream.cork();
        });
        stream.on('job', (job) => {
            job
                .on('error', err => {
                stream.destroy(err);
            })
                .on('complete', () => {
                stream.emit('complete', job);
                stream.uncork();
            });
        });
        return stream;
    }
    extract(destination, optionsOrCallback, cb) {
        const options = typeof optionsOrCallback === 'object' ? optionsOrCallback : {};
        const callback = typeof optionsOrCallback === 'function' ? optionsOrCallback : cb;
        this.createExtractJob(destination, options, (err, job, resp) => {
            if (err) {
                callback(err, resp);
                return;
            }
            job.on('error', callback).on('complete', metadata => {
                callback(null, metadata);
            });
        });
    }
    /**
     * Retrieves table data from a specified set of rows. The rows are returned to
     * your callback as an array of objects matching your table's schema.
     *
     * See {@link https://cloud.google.com/bigquery/docs/reference/v2/tabledata/list| Tabledata: list API Documentation}
     *
     * @param {object} [options] The configuration object.
     * @param {boolean} [options.autoPaginate=true] Have pagination handled
     *     automatically.
     * @param {number} [options.maxApiCalls] Maximum number of API calls to make.
     * @param {number} [options.maxResults] Maximum number of results to return.
     * @param {boolean|IntegerTypeCastOptions} [options.wrapIntegers=false] Wrap values
     *     of 'INT64' type in {@link BigQueryInt} objects.
     *     If a `boolean`, this will wrap values in {@link BigQueryInt} objects.
     *     If an `object`, this will return a value returned by
     *     `wrapIntegers.integerTypeCastFunction`.
     * @param {RowsCallback} [callback] The callback function. If `autoPaginate`
     *     is set to false a {@link ManualQueryResultsCallback} should be used.
     * @param {?error} callback.err An error returned while making this request
     * @param {array} callback.rows The table data from specified set of rows.
     * @returns {Promise<RowsResponse>}
     *
     * @example
     * ```
     * const {BigQuery} = require('@google-cloud/bigquery');
     * const bigquery = new BigQuery();
     * const dataset = bigquery.dataset('my-dataset');
     * const table = dataset.table('my-table');
     *
     * table.getRows((err, rows) => {
     *   if (!err) {
     *     // rows is an array of results.
     *   }
     * });
     *
     * //-
     * // To control how many API requests are made and page through the results
     * // manually, set `autoPaginate` to `false`.
     * //-
     * function manualPaginationCallback(err, rows, nextQuery, apiResponse) {
     *   if (nextQuery) {
     *     // More results exist.
     *     table.getRows(nextQuery, manualPaginationCallback);
     *   }
     * }
     *
     * table.getRows({
     *   autoPaginate: false
     * }, manualPaginationCallback);
     *
     * //-
     * // If the callback is omitted, we'll return a Promise.
     * //-
     * table.getRows().then((data) => {
     *   const rows = data[0];
     *   });
     * ```
     */
    getRows(optionsOrCallback, cb) {
        const options = typeof optionsOrCallback === 'object' ? optionsOrCallback : {};
        const callback = typeof optionsOrCallback === 'function' ? optionsOrCallback : cb;
        const wrapIntegers = options.wrapIntegers ? options.wrapIntegers : false;
        delete options.wrapIntegers;
        const parseJSON = options.parseJSON ? options.parseJSON : false;
        delete options.parseJSON;
        const onComplete = (err, rows, nextQuery, resp) => {
            if (err) {
                callback(err, null, null, resp);
                return;
            }
            rows = _1.BigQuery.mergeSchemaWithRows_(this.metadata.schema, rows || [], {
                wrapIntegers: wrapIntegers,
                selectedFields: options.selectedFields
                    ? options.selectedFields.split(',')
                    : [],
                parseJSON,
            });
            callback(null, rows, nextQuery, resp);
        };
        const qs = extend({
            'formatOptions.useInt64Timestamp': true,
        }, options);
        this.request({
            uri: '/data',
            qs,
        }, (err, resp) => {
            if (err) {
                onComplete(err, null, null, resp);
                return;
            }
            let nextQuery = null;
            if (resp.pageToken) {
                nextQuery = Object.assign({}, qs, {
                    pageToken: resp.pageToken,
                });
            }
            if (resp.rows && resp.rows.length > 0 && !this.metadata.schema) {
                // We don't know the schema for this table yet. Do a quick stat.
                this.getMetadata((err, metadata, apiResponse) => {
                    if (err) {
                        onComplete(err, null, null, apiResponse);
                        return;
                    }
                    onComplete(null, resp.rows, nextQuery, resp);
                });
                return;
            }
            onComplete(null, resp.rows, nextQuery, resp);
        });
    }
    insert(rows, optionsOrCallback, cb) {
        const options = typeof optionsOrCallback === 'object'
            ? optionsOrCallback
            : {};
        const callback = typeof optionsOrCallback === 'function' ? optionsOrCallback : cb;
        const promise = this._insertAndCreateTable(rows, options);
        if (callback) {
            promise.then(resp => callback(null, resp), err => callback(err, null));
        }
        else {
            return promise.then(r => [r]);
        }
    }
    /**
     * Insert rows with retries, but will create the table if not exists.
     *
     * @param {RowMetadata | RowMetadata[]} rows
     * @param {InsertRowsOptions} options
     * @returns {Promise<bigquery.ITableDataInsertAllResponse | bigquery.ITable>}
     * @private
     */
    async _insertAndCreateTable(rows, options) {
        const { schema } = options;
        const delay = 60000;
        try {
            return await this._insertWithRetry(rows, options);
        }
        catch (err) {
            if (err.code !== 404 || !schema) {
                throw err;
            }
        }
        try {
            await this.create({ schema });
        }
        catch (err) {
            if (err.code !== 409) {
                throw err;
            }
        }
        // table creation after failed access is subject to failure caching and
        // eventual consistency, see:
        // https://github.com/googleapis/google-cloud-python/issues/4553#issuecomment-350110292
        await new Promise(resolve => setTimeout(resolve, delay));
        return this._insertAndCreateTable(rows, options);
    }
    /**
     * This method will attempt to insert rows while retrying any partial failures
     * that occur along the way. Because partial insert failures are returned
     * differently, we can't depend on our usual retry strategy.
     *
     * @private
     *
     * @param {RowMetadata|RowMetadata[]} rows The rows to insert.
     * @param {InsertRowsOptions} options Insert options.
     * @returns {Promise<bigquery.ITableDataInsertAllResponse>}
     */
    async _insertWithRetry(rows, options) {
        const { partialRetries = 3 } = options;
        let error;
        const maxAttempts = Math.max(partialRetries, 0) + 1;
        for (let attempts = 0; attempts < maxAttempts; attempts++) {
            try {
                return await this._insert(rows, options);
            }
            catch (e) {
                error = e;
                rows = (e.errors || [])
                    .filter(err => !!err.row)
                    .map(err => err.row);
                if (!rows.length) {
                    break;
                }
            }
        }
        throw error;
    }
    /**
     * This method does the bulk of the work for processing options and making the
     * network request.
     *
     * @private
     *
     * @param {RowMetadata|RowMetadata[]} rows The rows to insert.
     * @param {InsertRowsOptions} options Insert options.
     * @returns {Promise<bigquery.ITableDataInsertAllResponse>}
     */
    async _insert(rows, options) {
        rows = arrify(rows);
        if (!rows.length) {
            throw new Error('You must provide at least 1 row to be inserted.');
        }
        const json = extend(true, {}, options, { rows });
        if (!options.raw) {
            json.rows = rows.map((row) => {
                const encoded = {
                    json: Table.encodeValue_(row),
                };
                if (options.createInsertId !== false) {
                    encoded.insertId = uuid.v4();
                }
                return encoded;
            });
        }
        delete json.createInsertId;
        delete json.partialRetries;
        delete json.raw;
        delete json.schema;
        const [resp] = await this.request({
            method: 'POST',
            uri: '/insertAll',
            json,
        });
        const partialFailures = (resp.insertErrors || []).map((insertError) => {
            return {
                errors: insertError.errors.map(error => {
                    return {
                        message: error.message,
                        reason: error.reason,
                    };
                }),
                // eslint-disable-next-line @typescript-eslint/no-explicit-any
                row: rows[insertError.index],
            };
        });
        if (partialFailures.length > 0) {
            throw new common_1.util.PartialFailureError({
                errors: partialFailures,
                response: resp,
            });
        }
        return resp;
    }
    createInsertStream(options) {
        options = typeof options === 'object' ? options : {};
        const dup = new stream_1.Duplex({ objectMode: true });
        dup._write = (chunk, encoding, cb) => {
            this.rowQueue.add(chunk, () => { });
            cb();
        };
        this.rowQueue = new rowQueue_1.RowQueue(this, dup, options);
        return dup;
    }
    load(source, metadataOrCallback, cb) {
        const metadata = typeof metadataOrCallback === 'object' ? metadataOrCallback : {};
        const callback = typeof metadataOrCallback === 'function' ? metadataOrCallback : cb;
        this.createLoadJob(source, metadata, (err, job, resp) => {
            if (err) {
                callback(err, resp);
                return;
            }
            job.on('error', callback).on('complete', metadata => {
                callback(null, metadata);
            });
        });
    }
    query(query, callback) {
        if (typeof query === 'string') {
            query = {
                query,
            };
        }
        this.dataset.query(query, callback);
    }
    setMetadata(metadata, callback) {
        const body = Table.formatMetadata_(metadata);
        super.setMetadata(body, callback);
    }
    getIamPolicy(optionsOrCallback, cb) {
        const options = typeof optionsOrCallback === 'object' ? optionsOrCallback : {};
        const callback = typeof optionsOrCallback === 'function' ? optionsOrCallback : cb;
        if (typeof options.requestedPolicyVersion === 'number' &&
            options.requestedPolicyVersion !== 1) {
            throw new Error('Only IAM policy version 1 is supported.');
        }
        const json = extend(true, {}, { options });
        this.request({
            method: 'POST',
            uri: '/:getIamPolicy',
            json,
        }, (err, resp) => {
            if (err) {
                callback(err, null);
                return;
            }
            callback(null, resp);
        });
    }
    setIamPolicy(policy, optionsOrCallback, cb) {
        const options = typeof optionsOrCallback === 'object' ? optionsOrCallback : {};
        const callback = typeof optionsOrCallback === 'function' ? optionsOrCallback : cb;
        if (policy.version && policy.version !== 1) {
            throw new Error('Only IAM policy version 1 is supported.');
        }
        const json = extend(true, {}, options, { policy });
        this.request({
            method: 'POST',
            uri: '/:setIamPolicy',
            json,
        }, (err, resp) => {
            if (err) {
                callback(err, null);
                return;
            }
            callback(null, resp);
        });
    }
    testIamPermissions(permissions, callback) {
        permissions = arrify(permissions);
        const json = extend(true, {}, { permissions });
        this.request({
            method: 'POST',
            uri: '/:testIamPermissions',
            json,
        }, (err, resp) => {
            if (err) {
                callback(err, null);
                return;
            }
            callback(null, resp);
        });
    }
}
exports.Table = Table;
/*! Developer Documentation
 *
 * These methods can be auto-paginated.
 */
paginator_1.paginator.extend(Table, ['getRows']);
/*! Developer Documentation
 *
 * All async methods (except for streams) will return a Promise in the event
 * that a callback is omitted.
 */
(0, promisify_1.promisifyAll)(Table);
//# sourceMappingURL=table.js.map