{
  "name": "Source -> KG (idempotent sourcing)",
  "goal": "Find entities and write them to a KG list with a stable userKey so re-running never produces duplicates.",
  "description": "Pattern 15a (sourcing half of the Source -> Process split). Pair this scaffold with `kg-process-update` as the consumer. Uses `kg.upsert-rows` with `userKey` for O(1) dedup across runs and `status: \"new\"` so the processing workflow can filter cleanly.",
  "status": "draft",
  "context": {
    "executionInputConfig": {
      "title": "Source Entities",
      "description": "Find entities and add them to the shared KG list.",
      "runCTALabel": "Source",
      "allowRegeneration": false,
      "fields": [
        { "name": "search_query", "label": "Search query / topic", "type": "text", "required": true },
        { "name": "target_list_key", "label": "Target KG list key", "type": "text", "required": true }
      ]
    }
  },
  "steps": [
    {
      "id": "schedule",
      "type": "trigger",
      "name": "Weekly source",
      "pipelineStepStartConditions": {
        "trigger": {
          "type": "schedule",
          "description": "Run weekly on Monday morning.",
          "config": { "frequency": "weekly", "dayOfWeek": "monday", "time": "08:00" }
        }
      },
      "next": { "stepId": "generate-queries" }
    },
    {
      "id": "generate-queries",
      "type": "aiAction",
      "name": "Generate Optimized Queries",
      "pipelineStepPrompt": {
        "template": "Generate 3 optimized search queries to find new entities matching: {{input.search_query}}\n\nUse boolean operators and filters appropriate to the target platform. Return a flat list of strings.",
        "responseStructure": {
          "queries": "array of strings"
        }
      },
      "creditCost": 3,
      "next": { "stepId": "search" }
    },
    {
      "id": "search",
      "type": "appAction",
      "name": "Search (replace with target platform)",
      "app": { "id": "REPLACE_WITH_APP_ID", "actionId": "REPLACE_WITH_APP_ID.REPLACE_WITH_ACTION_ID", "source": "native" },
      "stepInputData": {
        "queries": "{{steps.generate-queries.queries}}"
      },
      "creditCost": 2,
      "next": { "stepId": "extract" }
    },
    {
      "id": "extract",
      "type": "aiAction",
      "name": "Extract Entities",
      "pipelineStepPrompt": {
        "template": "Extract structured entity records from this raw source output.\n\nSource: {{steps.search.output}}\n\nEvery row MUST include a stable identifier (canonical URL, domain, or platform id) — this becomes the userKey for KG dedup. Drop rows you cannot uniquely identify.",
        "responseStructure": {
          "items": "array of { userKey: string (stable id e.g. canonical URL), rowData: { name, url, description, ...other fields } }"
        }
      },
      "creditCost": 5,
      "next": { "stepId": "save-to-kg" }
    },
    {
      "id": "save-to-kg",
      "type": "appAction",
      "name": "Upsert to KG (status: new)",
      "app": { "id": "kg", "actionId": "kg.upsert-rows", "source": "native" },
      "stepInputData": {
        "listKey": "{{input.target_list_key}}",
        "rows": "{{steps.extract.items}}",
        "mergeStrategy": "merge",
        "status": "new"
      },
      "next": { "stepId": "done" }
    },
    {
      "id": "done",
      "type": "milestone",
      "name": "Sourcing Done"
    }
  ]
}