@azure/storage-blob/dist-esm/storage-internal-avro/src/AvroReader.js.map

{"version":3,"file":"AvroReader.js","sourceRoot":"","sources":["../../../../storage-internal-avro/src/AvroReader.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,kCAAkC;;AAElC,sCAAsC;AACtC,iEAAiE;AAEjE,OAAO,oBAAoB,CAAC;AAC5B,OAAO,EACL,cAAc,EACd,eAAe,EACf,eAAe,EACf,qBAAqB,GACtB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAGpD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAanD,MAAM,OAAO,UAAU;IAuCrB,YACE,UAAwB,EACxB,YAA2B,EAC3B,kBAA2B,EAC3B,uBAAgC;QAEhC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,IAAI,CAAC,aAAa,GAAG,YAAY,IAAI,UAAU,CAAC;QAChD,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,YAAY,GAAG,kBAAkB,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,YAAY,GAAG,uBAAuB,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,mBAAmB,GAAG,kBAAkB,IAAI,CAAC,CAAC;IACrD,CAAC;IAhCD,IAAW,WAAW;QACpB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAGD,IAAW,WAAW;QACpB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IA2BO,KAAK,CAAC,UAAU,CAAC,UAA4B,EAAE;QACrD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,cAAc,CAAC,IAAI,CAAC,aAAa,EAAE,eAAe,CAAC,MAAM,EAAE;YACzF,WAAW,EAAE,OAAO,CAAC,WAAW;SACjC,CAAC,CAAC;QACH,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,eAAe,CAAC,EAAE;YACzC,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;SAChD;QAED,sEAAsE;QACtE,sCAAsC;QACtC,IAAI,CAAC,SAAS,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE,UAAU,CAAC,UAAU,EAAE;YACnF,WAAW,EAAE,OAAO,CAAC,WAAW;SACjC,CAAC,CAAC;QAEH,iBAAiB;QACjB,MAAM,KAAK,GAAG,IAAI,CAAC,SAAU,CAAC,cAAc,CAAC,CAAC;QAC9C,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,MAAM,CAAC,EAAE;YAChE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;SAC7C;QAED,6DAA6D;QAC7D,IAAI,CAAC,WAAW,GAAG,MAAM,UAAU,CAAC,cAAc,CAAC,IAAI,CAAC,aAAa,EAAE,qBAAqB,EAAE;YAC5F,WAAW,EAAE,OAAO,CAAC,WAAW;SACjC,CAAC,CAAC;QAEH,mBAAmB;QACnB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAU,CAAC,eAAe,CAAC,CAAC,CAAC;QAC5D,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAE7C,IAAI,IAAI,CAAC,YAAY,KAAK,CAAC,EAAE;YAC3B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;SAC1E;QAED,IAAI,CAAC,sBAAsB,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE;YACxE,WAAW,EAAE,OAAO,CAAC,WAAW;SACjC,CAAC,CAAC;QACH,oBAAoB;QACpB,MAAM,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;QAElF,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,GAAG,CAAC,EAAE;YAC9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC,EAAE,EAAE;gBAC1C,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;gBAClF,IAAI,CAAC,sBAAuB,EAAE,CAAC;aAChC;SACF;IACH,CAAC;IAEM,OAAO;QACZ,OAAO,CAAC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,sBAAuB,GAAG,CAAC,CAAC;IAChE,CAAC;IAEa,YAAY,CACxB,UAA4B,EAAE;;YAE9B,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;gBACtB,cAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAA,CAAC;aAChC;YAED,OAAO,IAAI,CAAC,OAAO,EAAE,EAAE;gBACrB,MAAM,MAAM,GAAG,cAAM,IAAI,CAAC,SAAU,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;oBAC1D,WAAW,EAAE,OAAO,CAAC,WAAW;iBACjC,CAAC,CAAA,CAAC;gBAEH,IAAI,CAAC,sBAAuB,EAAE,CAAC;gBAC/B,IAAI,CAAC,YAAa,EAAE,CAAC;gBAErB,IAAI,IAAI,CAAC,sBAAsB,KAAK,CAAC,EAAE;oBACrC,MAAM,MAAM,GAAG,cAAM,UAAU,CAAC,cAAc,CAAC,IAAI,CAAC,WAAW,EAAE,qBAAqB,EAAE;wBACtF,WAAW,EAAE,OAAO,CAAC,WAAW;qBACjC,CAAC,CAAA,CAAC;oBAEH,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;oBACzE,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;oBAEtB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,WAAY,EAAE,MAAM,CAAC,EAAE;wBAC3C,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;qBACrD;oBAED,IAAI;wBACF,IAAI,CAAC,sBAAsB,GAAG,cAAM,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE;4BACxE,WAAW,EAAE,OAAO,CAAC,WAAW;yBACjC,CAAC,CAAA,CAAC;qBACJ;oBAAC,OAAO,GAAQ,EAAE;wBACjB,gCAAgC;wBAChC,IAAI,CAAC,sBAAsB,GAAG,CAAC,CAAC;qBACjC;oBAED,IAAI,IAAI,CAAC,sBAAuB,GAAG,CAAC,EAAE;wBACpC,oBAAoB;wBACpB,cAAM,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAA,CAAC;qBACnF;iBACF;gBACD,oBAAM,MAAM,CAAA,CAAC;aACd;QACH,CAAC;KAAA;CACF","sourcesContent":["// Copyright (c) Microsoft Corporation.\n// Licensed under the MIT license.\n\n// TODO: Do a review of non-interfaces\n/* eslint-disable @azure/azure-sdk/ts-use-interface-parameters */\n\nimport \"@azure/core-paging\";\nimport {\n  AVRO_CODEC_KEY,\n  AVRO_INIT_BYTES,\n  AVRO_SCHEMA_KEY,\n  AVRO_SYNC_MARKER_SIZE,\n} from \"./AvroConstants\";\nimport { AvroParser, AvroType } from \"./AvroParser\";\nimport { AbortSignalLike } from \"@azure/abort-controller\";\nimport { AvroReadable } from \"./AvroReadable\";\nimport { arraysEqual } from \"./utils/utils.common\";\n\n/**\n * Options to configure the {@link AvroReader.parseObjects} operation.\n */\nexport interface AvroParseOptions {\n  /**\n   * An implementation of the `AbortSignalLike` interface to signal the request to cancel the operation.\n   * For example, use the &commat;azure/abort-controller to create an `AbortSignal`.\n   */\n  abortSignal?: AbortSignalLike;\n}\n\nexport class AvroReader {\n  private readonly _dataStream: AvroReadable;\n\n  private readonly _headerStream: AvroReadable;\n\n  private _syncMarker?: Uint8Array;\n\n  private _metadata?: Record<string, string>;\n\n  private _itemType?: AvroType;\n\n  private _itemsRemainingInBlock?: number;\n\n  // Remembers where we started if partial data stream was provided.\n  private readonly _initialBlockOffset: number;\n\n  /// The byte offset within the Avro file (both header and data)\n  /// of the start of the current block.\n  private _blockOffset: number;\n  public get blockOffset(): number {\n    return this._blockOffset;\n  }\n\n  private _objectIndex: number;\n  public get objectIndex(): number {\n    return this._objectIndex;\n  }\n\n  private _initialized: boolean;\n\n  constructor(dataStream: AvroReadable);\n\n  constructor(\n    dataStream: AvroReadable,\n    headerStream: AvroReadable,\n    currentBlockOffset: number,\n    indexWithinCurrentBlock: number\n  );\n\n  constructor(\n    dataStream: AvroReadable,\n    headerStream?: AvroReadable,\n    currentBlockOffset?: number,\n    indexWithinCurrentBlock?: number\n  ) {\n    this._dataStream = dataStream;\n    this._headerStream = headerStream || dataStream;\n    this._initialized = false;\n    this._blockOffset = currentBlockOffset || 0;\n    this._objectIndex = indexWithinCurrentBlock || 0;\n    this._initialBlockOffset = currentBlockOffset || 0;\n  }\n\n  private async initialize(options: AvroParseOptions = {}): Promise<void> {\n    const header = await AvroParser.readFixedBytes(this._headerStream, AVRO_INIT_BYTES.length, {\n      abortSignal: options.abortSignal,\n    });\n    if (!arraysEqual(header, AVRO_INIT_BYTES)) {\n      throw new Error(\"Stream is not an Avro file.\");\n    }\n\n    // File metadata is written as if defined by the following map schema:\n    // { \"type\": \"map\", \"values\": \"bytes\"}\n    this._metadata = await AvroParser.readMap(this._headerStream, AvroParser.readString, {\n      abortSignal: options.abortSignal,\n    });\n\n    // Validate codec\n    const codec = this._metadata![AVRO_CODEC_KEY];\n    if (!(codec === undefined || codec === null || codec === \"null\")) {\n      throw new Error(\"Codecs are not supported\");\n    }\n\n    // The 16-byte, randomly-generated sync marker for this file.\n    this._syncMarker = await AvroParser.readFixedBytes(this._headerStream, AVRO_SYNC_MARKER_SIZE, {\n      abortSignal: options.abortSignal,\n    });\n\n    // Parse the schema\n    const schema = JSON.parse(this._metadata![AVRO_SCHEMA_KEY]);\n    this._itemType = AvroType.fromSchema(schema);\n\n    if (this._blockOffset === 0) {\n      this._blockOffset = this._initialBlockOffset + this._dataStream.position;\n    }\n\n    this._itemsRemainingInBlock = await AvroParser.readLong(this._dataStream, {\n      abortSignal: options.abortSignal,\n    });\n    // skip block length\n    await AvroParser.readLong(this._dataStream, { abortSignal: options.abortSignal });\n\n    this._initialized = true;\n    if (this._objectIndex && this._objectIndex > 0) {\n      for (let i = 0; i < this._objectIndex; i++) {\n        await this._itemType.read(this._dataStream, { abortSignal: options.abortSignal });\n        this._itemsRemainingInBlock!--;\n      }\n    }\n  }\n\n  public hasNext(): boolean {\n    return !this._initialized || this._itemsRemainingInBlock! > 0;\n  }\n\n  public async *parseObjects(\n    options: AvroParseOptions = {}\n  ): AsyncIterableIterator<Record<string, any> | null> {\n    if (!this._initialized) {\n      await this.initialize(options);\n    }\n\n    while (this.hasNext()) {\n      const result = await this._itemType!.read(this._dataStream, {\n        abortSignal: options.abortSignal,\n      });\n\n      this._itemsRemainingInBlock!--;\n      this._objectIndex!++;\n\n      if (this._itemsRemainingInBlock === 0) {\n        const marker = await AvroParser.readFixedBytes(this._dataStream, AVRO_SYNC_MARKER_SIZE, {\n          abortSignal: options.abortSignal,\n        });\n\n        this._blockOffset = this._initialBlockOffset + this._dataStream.position;\n        this._objectIndex = 0;\n\n        if (!arraysEqual(this._syncMarker!, marker)) {\n          throw new Error(\"Stream is not a valid Avro file.\");\n        }\n\n        try {\n          this._itemsRemainingInBlock = await AvroParser.readLong(this._dataStream, {\n            abortSignal: options.abortSignal,\n          });\n        } catch (err: any) {\n          // We hit the end of the stream.\n          this._itemsRemainingInBlock = 0;\n        }\n\n        if (this._itemsRemainingInBlock! > 0) {\n          // Ignore block size\n          await AvroParser.readLong(this._dataStream, { abortSignal: options.abortSignal });\n        }\n      }\n      yield result;\n    }\n  }\n}\n"]}