# Transform Jobs

## List Transform Jobs

> List transform jobs for the authenticated organization

```json
{"openapi":"3.1.0","info":{"title":"LightningRod API","version":"1.0.0"},"servers":[{"url":"/api/public/v1"}],"security":[{"HTTPBearer":[]}],"components":{"securitySchemes":{"HTTPBearer":{"type":"http","scheme":"bearer"}},"schemas":{"TransformJobStatus":{"type":"string","enum":["RUNNING","COMPLETED","FAILED","CANCELLED"],"title":"TransformJobStatus"},"ListTransformJobsResponse":{"properties":{"jobs":{"items":{"$ref":"#/components/schemas/TransformJob"},"type":"array","title":"Jobs"},"next_cursor":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Next Cursor"},"has_more":{"type":"boolean","title":"Has More"},"total":{"type":"integer","title":"Total"}},"type":"object","required":["jobs","has_more","total"],"title":"ListTransformJobsResponse"},"TransformJob":{"properties":{"id":{"type":"string","title":"Id"},"organization_id":{"type":"string","title":"Organization Id"},"configuration_id":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Configuration Id"},"name":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Name"},"status":{"$ref":"#/components/schemas/TransformJobStatus"},"error_message":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Error Message"},"warning_message":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Warning Message"},"modal_function_call_id":{"type":"string","title":"Modal Function Call Id"},"modal_app_id":{"type":"string","title":"Modal App Id"},"transform_config":{"type":"string","title":"Transform Config"},"input_dataset_id":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Input Dataset Id"},"output_dataset_id":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Output Dataset Id"},"usage":{"anyOf":[{"$ref":"#/components/schemas/JobUsage"},{"type":"null"}]},"estimated_cost_dollars":{"anyOf":[{"type":"number"},{"type":"null"}],"title":"Estimated Cost Dollars"},"created_at":{"type":"string","format":"date-time","title":"Created At"},"updated_at":{"type":"string","format":"date-time","title":"Updated At"}},"type":"object","required":["id","organization_id","status","modal_function_call_id","modal_app_id","transform_config","input_dataset_id","output_dataset_id","created_at","updated_at"],"title":"TransformJob"},"JobUsage":{"properties":{"total":{"anyOf":[{"$ref":"#/components/schemas/UsageSummary"},{"type":"null"}]},"by_step":{"anyOf":[{"additionalProperties":{"$ref":"#/components/schemas/UsageSummary"},"type":"object"},{"type":"null"}],"title":"By Step"},"max_cost_dollars":{"anyOf":[{"type":"number"},{"type":"null"}],"title":"Max Cost Dollars"},"current_cost_dollars":{"anyOf":[{"type":"number"},{"type":"null"}],"title":"Current Cost Dollars"},"estimated_cost_dollars":{"anyOf":[{"type":"number"},{"type":"null"}],"title":"Estimated Cost Dollars"},"pipeline_summary":{"anyOf":[{"items":{"$ref":"#/components/schemas/PipelineStepSummary"},"type":"array"},{"type":"null"}],"title":"Pipeline Summary"}},"type":"object","title":"JobUsage","description":"Aggregated usage statistics for a transform job with step breakdown."},"UsageSummary":{"properties":{"events":{"additionalProperties":{"$ref":"#/components/schemas/EventUsageSummary"},"type":"object","title":"Events","default":{}},"llm_by_model":{"additionalProperties":{"$ref":"#/components/schemas/LLMModelUsageSummary"},"type":"object","title":"Llm By Model","default":{}},"total_cost":{"type":"number","title":"Total Cost","default":0}},"type":"object","title":"UsageSummary","description":"Flexible usage statistics by event type and LLM model."},"EventUsageSummary":{"properties":{"count":{"type":"integer","title":"Count","default":0},"amount":{"type":"number","title":"Amount","default":0},"cost":{"type":"number","title":"Cost","default":0},"cached_count":{"type":"integer","title":"Cached Count","default":0}},"type":"object","title":"EventUsageSummary","description":"Usage for a single event type."},"LLMModelUsageSummary":{"properties":{"count":{"type":"integer","title":"Count","default":0},"input_tokens":{"type":"integer","title":"Input Tokens","default":0},"output_tokens":{"type":"integer","title":"Output Tokens","default":0},"cost":{"type":"number","title":"Cost","default":0},"cached_count":{"type":"integer","title":"Cached Count","default":0}},"type":"object","title":"LLMModelUsageSummary","description":"Usage for a single LLM model."},"PipelineStepSummary":{"properties":{"step_index":{"type":"integer","title":"Step Index"},"transform_name":{"type":"string","title":"Transform Name"},"input_count":{"type":"integer","title":"Input Count","default":0},"output_count":{"type":"integer","title":"Output Count","default":0},"rejected_count":{"type":"integer","title":"Rejected Count","default":0},"error_count":{"type":"integer","title":"Error Count","default":0},"rejection_reasons":{"additionalProperties":{"type":"integer"},"type":"object","title":"Rejection Reasons","default":{}}},"type":"object","required":["step_index","transform_name"],"title":"PipelineStepSummary","description":"Summary of a single pipeline step including rejection reasons."},"HTTPValidationError":{"properties":{"detail":{"items":{"$ref":"#/components/schemas/ValidationError"},"type":"array","title":"Detail"}},"type":"object","title":"HTTPValidationError"},"ValidationError":{"properties":{"loc":{"items":{"anyOf":[{"type":"string"},{"type":"integer"}]},"type":"array","title":"Location"},"msg":{"type":"string","title":"Message"},"type":{"type":"string","title":"Error Type"},"input":{"title":"Input"},"ctx":{"type":"object","title":"Context"}},"type":"object","required":["loc","msg","type"],"title":"ValidationError"}}},"paths":{"/transform-jobs":{"get":{"tags":["Transform Jobs"],"summary":"List Transform Jobs","description":"List transform jobs for the authenticated organization","operationId":"list_transform_jobs_transform_jobs_get","parameters":[{"name":"limit","in":"query","required":false,"schema":{"type":"integer","maximum":100,"minimum":1,"default":10,"title":"Limit"}},{"name":"cursor","in":"query","required":false,"schema":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Cursor"}},{"name":"status","in":"query","required":false,"schema":{"anyOf":[{"$ref":"#/components/schemas/TransformJobStatus"},{"type":"null"}],"title":"Status"}},{"name":"configuration_id","in":"query","required":false,"schema":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Configuration Id"}}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ListTransformJobsResponse"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}}}}
```

## Create Transform Job

> Create a new transform job

```json
{"openapi":"3.1.0","info":{"title":"LightningRod API","version":"1.0.0"},"servers":[{"url":"/api/public/v1"}],"security":[{"HTTPBearer":[]}],"components":{"securitySchemes":{"HTTPBearer":{"type":"http","scheme":"bearer"}},"schemas":{"CreateTransformJobRequest":{"properties":{"config":{"$ref":"#/components/schemas/CreateTransformConfig"},"input_dataset_id":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Input Dataset Id"},"max_questions":{"anyOf":[{"type":"integer"},{"type":"null"}],"title":"Max Questions"},"max_cost_dollars":{"anyOf":[{"type":"number"},{"type":"null"}],"title":"Max Cost Dollars"},"configuration_id":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Configuration Id"},"name":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Name"}},"type":"object","required":["config"],"title":"CreateTransformJobRequest"},"CreateTransformConfig":{"oneOf":[{"$ref":"#/components/schemas/ForwardLookingQuestionGenerator"},{"$ref":"#/components/schemas/FileSetQuerySeedGenerator"},{"$ref":"#/components/schemas/FileSetSeedGenerator"},{"$ref":"#/components/schemas/GdeltSeedGenerator"},{"$ref":"#/components/schemas/NewsSeedGenerator"},{"$ref":"#/components/schemas/QuestionAndLabelGenerator"},{"$ref":"#/components/schemas/QuestionGenerator"},{"$ref":"#/components/schemas/QuestionPipeline"},{"$ref":"#/components/schemas/QuestionRenderer"},{"$ref":"#/components/schemas/WebSearchLabeler"}],"discriminator":{"propertyName":"config_type","mapping":{"FILESET_QUERY_SEED_GENERATOR":"#/components/schemas/FileSetQuerySeedGenerator","FILESET_SEED_GENERATOR":"#/components/schemas/FileSetSeedGenerator","FORWARD_LOOKING_QUESTION_GENERATOR":"#/components/schemas/ForwardLookingQuestionGenerator","GDELT_SEED_GENERATOR":"#/components/schemas/GdeltSeedGenerator","NEWS_SEED_GENERATOR":"#/components/schemas/NewsSeedGenerator","QUESTION_AND_LABEL_GENERATOR":"#/components/schemas/QuestionAndLabelGenerator","QUESTION_GENERATOR":"#/components/schemas/QuestionGenerator","QUESTION_PIPELINE":"#/components/schemas/QuestionPipeline","QUESTION_RENDERER":"#/components/schemas/QuestionRenderer","WEB_SEARCH_LABELER":"#/components/schemas/WebSearchLabeler"}}},"ForwardLookingQuestionGenerator":{"properties":{"config_type":{"type":"string","const":"FORWARD_LOOKING_QUESTION_GENERATOR","title":"Config Type","description":"Type of transform configuration","default":"FORWARD_LOOKING_QUESTION_GENERATOR"},"instructions":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Instructions","description":"Instructions for question generation. If not provided, uses sensible defaults."},"bad_examples":{"items":{"type":"string"},"type":"array","title":"Bad Examples","description":"Examples of questions to avoid"},"filter":{"anyOf":[{"$ref":"#/components/schemas/FilterCriteria"},{"items":{"$ref":"#/components/schemas/FilterCriteria"},"type":"array"},{"type":"null"}],"title":"Filter","description":"Optional filter criteria to apply after question generation"},"questions_per_seed":{"type":"integer","title":"Questions Per Seed","description":"Number of questions to generate per seed","default":1},"include_default_filter":{"type":"boolean","title":"Include Default Filter","description":"Whether to include the default filter for generated questions","default":false},"answer_type":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/BinaryAnswerType"},{"$ref":"#/components/schemas/MultipleChoiceAnswerType"},{"$ref":"#/components/schemas/ContinuousAnswerType"},{"$ref":"#/components/schemas/FreeResponseAnswerType"}],"discriminator":{"propertyName":"answer_type","mapping":{"BINARY":"#/components/schemas/BinaryAnswerType","CONTINUOUS":"#/components/schemas/ContinuousAnswerType","FREE_RESPONSE":"#/components/schemas/FreeResponseAnswerType","MULTIPLE_CHOICE":"#/components/schemas/MultipleChoiceAnswerType"}}},{"type":"null"}],"title":"Answer Type","description":"The type of answer expected for generated questions"}},"type":"object","title":"ForwardLookingQuestionGenerator"},"FilterCriteria":{"properties":{"rubric":{"type":"string","title":"Rubric","description":"Scoring rubric/prompt for evaluating content"},"min_score":{"type":"number","maximum":1,"minimum":0,"title":"Min Score","description":"Minimum score threshold","default":0.5},"model_name":{"type":"string","title":"Model Name","description":"Name of the model (in openrouter) to use for scoring","default":"google/gemini-3-flash-preview"}},"type":"object","required":["rubric"],"title":"FilterCriteria","description":"Reusable filter criteria for LLM-based content scoring and filtering."},"BinaryAnswerType":{"properties":{"answer_type":{"type":"string","const":"BINARY","title":"Answer Type","default":"BINARY"},"answer_format_instruction":{"type":"string","title":"Answer Format Instruction","description":"Instructions describing how the answer should be formatted and given.","default":"This is a binary yes/no question. You are estimating the probability that the answer is 'Yes'. Provide your confidence as a value between 0 (definitely No) and 1 (definitely Yes). Provide your probability estimate for Yes as a decimal between 0 and 1. Provide your answer between <answer></answer> tags. Example: <answer>0.75</answer>"},"labeler_instruction":{"type":"string","title":"Labeler Instruction","description":"Instructions for the labeler.","default":"The answer should be ONLY '1', '0', or 'Undetermined'. '1' means yes, '0' means no, and 'Undetermined' means the answer is not clear. Do not include any other text or explanation."},"question_generation_instruction":{"type":"string","title":"Question Generation Instruction","description":"Instructions for generating questions of this type.","default":"Generate questions that can be answered with Yes or No. Frame the question so it has exactly two possible outcomes: Yes or No. The question should start with words like 'Will', 'Is', 'Does', 'Has', 'Can', etc. A clear and unambiguous yes/no question, based on the provided seed_text."},"reward_function_type":{"anyOf":[{"$ref":"#/components/schemas/RewardFunctionType"},{"type":"null"}],"default":"binary_brier"}},"type":"object","title":"BinaryAnswerType"},"RewardFunctionType":{"type":"string","enum":["binary_brier","binary_weighted_brier","binary_log_score","multi_choice_log_score","continuous_log_score","continuous_value_only_log_score","continuous_value_only_linear_score"],"title":"RewardFunctionType"},"MultipleChoiceAnswerType":{"properties":{"answer_type":{"type":"string","const":"MULTIPLE_CHOICE","title":"Answer Type","default":"MULTIPLE_CHOICE"},"answer_format_instruction":{"type":"string","title":"Answer Format Instruction","description":"Instructions describing how the answer should be formatted and given.","default":"This is a multiple choice question with answer options. The list of options will be provided in the question. You are estimating the probability for each option being the correct answer. Provide your confidence for each option as a value between 0 and 1, where the probabilities must sum to 1. Provide your answer as a JSON dictionary between <answer></answer> tags, with keys like option_0, option_1, etc. corresponding to each option in order. Example: <answer>{\"option_0\": 0.3, \"option_1\": 0.4, \"option_2\": 0.2, \"option_3\": 0.1}</answer>"},"labeler_instruction":{"type":"string","title":"Labeler Instruction","description":"Instructions for the labeler.","default":"The answer should be ONLY one of the following options: 'A', 'B', 'C', or 'D', or 'Undetermined'. Do not include any other text or explanation."},"question_generation_instruction":{"type":"string","title":"Question Generation Instruction","description":"Instructions for generating questions of this type.","default":"Generate questions with multiple choice options (up to 10). Format the options on separate lines after the question:\nA) First option\nB) Second option\nC) Third option\n... and so on\n\nEach option should be distinct and mutually exclusive. A clear and unambiguous multiple-choice question, based on the provided seed_text. Include the options in the question text."},"reward_function_type":{"anyOf":[{"$ref":"#/components/schemas/RewardFunctionType"},{"type":"null"}],"default":"multi_choice_log_score"}},"type":"object","title":"MultipleChoiceAnswerType"},"ContinuousAnswerType":{"properties":{"answer_type":{"type":"string","const":"CONTINUOUS","title":"Answer Type","default":"CONTINUOUS"},"answer_format_instruction":{"type":"string","title":"Answer Format Instruction","description":"Instructions describing how the answer should be formatted and given.","default":"This question expects a numeric value as the answer. Provide your best estimate as a single number. Include units if specified in the question. Provide your answer between <answer></answer> tags. Example: <answer>42.5</answer>"},"labeler_instruction":{"type":"string","title":"Labeler Instruction","description":"Instructions for the labeler.","default":"The answer should be ONLY a single exact numeric value, not a range. For example: '42.5' or '1000', not '40-45' or 'between 900 and 1100', or 'Undetermined'. Do not include any other text or explanation."},"question_generation_instruction":{"type":"string","title":"Question Generation Instruction","description":"Instructions for generating questions of this type.","default":"Generate questions that expect a numeric value as the answer. Specify the units if applicable (e.g., dollars, percent, count). A clear and unambiguous question, based on the provided seed_text, that expects a numeric value as the answer."},"reward_function_type":{"anyOf":[{"$ref":"#/components/schemas/RewardFunctionType"},{"type":"null"}],"description":"Reward function type for scoring rollouts. None for answer types that don't support scoring."}},"type":"object","title":"ContinuousAnswerType"},"FreeResponseAnswerType":{"properties":{"answer_type":{"type":"string","const":"FREE_RESPONSE","title":"Answer Type","default":"FREE_RESPONSE"},"answer_format_instruction":{"type":"string","title":"Answer Format Instruction","description":"Instructions describing how the answer should be formatted and given.","default":"This question expects a free-form text response. Provide an answer that directly addresses what the question is asking. Provide your answer between <answer></answer> tags. Example: <answer>The company announced a new product line.</answer>"},"labeler_instruction":{"type":"string","title":"Labeler Instruction","description":"Instructions for the labeler.","default":"Respond with the correct answer as a text description."},"question_generation_instruction":{"type":"string","title":"Question Generation Instruction","description":"Instructions for generating questions of this type.","default":"Generate questions that expect a free-form text response. A clear and unambiguous question, based on the provided seed_text, that expects a free-form text response."},"reward_function_type":{"anyOf":[{"$ref":"#/components/schemas/RewardFunctionType"},{"type":"null"}],"description":"Reward function type for scoring rollouts. None for answer types that don't support scoring."}},"type":"object","title":"FreeResponseAnswerType"},"FileSetQuerySeedGenerator":{"properties":{"config_type":{"type":"string","const":"FILESET_QUERY_SEED_GENERATOR","title":"Config Type","description":"Type of transform configuration","default":"FILESET_QUERY_SEED_GENERATOR"},"file_set_id":{"type":"string","title":"File Set Id","description":"FileSet ID to query"},"prompts":{"items":{"type":"string"},"type":"array","title":"Prompts","description":"List of queries to run against the fileset"},"metadata_filters":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}],"title":"Metadata Filters","description":"Optional list of AIP-160 metadata filters to select which documents to process. Documents matching ANY filter will be included. (e.g., [\"ticker='AAL'\", \"ticker='MSFT'\"])"},"system_instruction":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"System Instruction","description":"Optional system instruction for the Gemini model"}},"type":"object","required":["file_set_id","prompts"],"title":"FileSetQuerySeedGenerator","description":"Configuration for FileSet Query Seed Generator transform."},"FileSetSeedGenerator":{"properties":{"config_type":{"type":"string","const":"FILESET_SEED_GENERATOR","title":"Config Type","description":"Type of transform configuration","default":"FILESET_SEED_GENERATOR"},"file_set_id":{"type":"string","title":"File Set Id","description":"FileSet ID to read files from"},"chunk_size":{"type":"integer","minimum":100,"title":"Chunk Size","description":"Number of characters per chunk","default":4000},"chunk_overlap":{"type":"integer","minimum":0,"title":"Chunk Overlap","description":"Number of overlapping characters between consecutive chunks","default":200},"metadata_filters":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}],"title":"Metadata Filters","description":"Optional list of metadata filters to select which files to process. Files matching ANY filter will be included. (e.g., [\"ticker='AAL'\", \"ticker='MSFT'\"])"}},"type":"object","required":["file_set_id"],"title":"FileSetSeedGenerator","description":"Configuration for FileSet Seed Generator transform."},"GdeltSeedGenerator":{"properties":{"config_type":{"type":"string","const":"GDELT_SEED_GENERATOR","title":"Config Type","description":"Type of transform configuration","default":"GDELT_SEED_GENERATOR"},"start_date":{"type":"string","format":"date-time","title":"Start Date","description":"Start date for seed search"},"end_date":{"type":"string","format":"date-time","title":"End Date","description":"End date for seed search"},"interval_duration_days":{"type":"integer","title":"Interval Duration Days","description":"Duration of each interval in days","default":7},"articles_per_interval":{"type":"integer","title":"Articles Per Interval","description":"Number of articles to fetch per interval from BigQuery","default":1000}},"type":"object","required":["start_date","end_date"],"title":"GdeltSeedGenerator"},"NewsSeedGenerator":{"properties":{"config_type":{"type":"string","const":"NEWS_SEED_GENERATOR","title":"Config Type","description":"Type of transform configuration","default":"NEWS_SEED_GENERATOR"},"start_date":{"type":"string","format":"date-time","title":"Start Date","description":"Start date for seed search"},"end_date":{"type":"string","format":"date-time","title":"End Date","description":"End date for seed search"},"interval_duration_days":{"type":"integer","title":"Interval Duration Days","description":"Duration of each interval in days","default":7},"search_query":{"anyOf":[{"type":"string"},{"items":{"type":"string"},"type":"array"}],"title":"Search Query","description":"Search query for news articles. If multiple queries are provided, a separate search will be done for each query."},"articles_per_search":{"type":"integer","title":"Articles Per Search","description":"Number of articles to fetch per search (max 100). Each query/domain combination is a separate search.","default":10},"filter_criteria":{"anyOf":[{"$ref":"#/components/schemas/FilterCriteria"},{"items":{"$ref":"#/components/schemas/FilterCriteria"},"type":"array"},{"type":"null"}],"title":"Filter Criteria","description":"Optional criteria for filtering news snippets before scraping"},"source_domain":{"anyOf":[{"type":"string"},{"items":{"type":"string"},"type":"array"},{"type":"null"}],"title":"Source Domain","description":"Optional URL source of the news articles, e.g. 'https://reuters.com/business', if multiple sources are provided, multiple searchers will be done for each interval"}},"type":"object","required":["start_date","end_date","search_query"],"title":"NewsSeedGenerator"},"QuestionAndLabelGenerator":{"properties":{"config_type":{"type":"string","const":"QUESTION_AND_LABEL_GENERATOR","title":"Config Type","description":"Type of transform configuration","default":"QUESTION_AND_LABEL_GENERATOR"},"instructions":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Instructions","description":"Instructions for question generation. If not provided, uses sensible defaults."},"bad_examples":{"items":{"type":"string"},"type":"array","title":"Bad Examples","description":"Examples of questions to avoid"},"filter":{"anyOf":[{"$ref":"#/components/schemas/FilterCriteria"},{"items":{"$ref":"#/components/schemas/FilterCriteria"},"type":"array"},{"type":"null"}],"title":"Filter","description":"Optional filter criteria to apply after question generation"},"questions_per_seed":{"type":"integer","title":"Questions Per Seed","description":"Number of questions to generate per seed","default":1},"include_default_filter":{"type":"boolean","title":"Include Default Filter","description":"Whether to include the default filter for generated questions","default":false},"answer_type":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/BinaryAnswerType"},{"$ref":"#/components/schemas/MultipleChoiceAnswerType"},{"$ref":"#/components/schemas/ContinuousAnswerType"},{"$ref":"#/components/schemas/FreeResponseAnswerType"}],"discriminator":{"propertyName":"answer_type","mapping":{"BINARY":"#/components/schemas/BinaryAnswerType","CONTINUOUS":"#/components/schemas/ContinuousAnswerType","FREE_RESPONSE":"#/components/schemas/FreeResponseAnswerType","MULTIPLE_CHOICE":"#/components/schemas/MultipleChoiceAnswerType"}}},{"type":"null"}],"title":"Answer Type","description":"The type of answer expected for generated questions"}},"type":"object","title":"QuestionAndLabelGenerator"},"QuestionGenerator":{"properties":{"config_type":{"type":"string","const":"QUESTION_GENERATOR","title":"Config Type","description":"Type of transform configuration","default":"QUESTION_GENERATOR"},"instructions":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Instructions","description":"Instructions for question generation. If not provided, uses sensible defaults."},"bad_examples":{"items":{"type":"string"},"type":"array","title":"Bad Examples","description":"Examples of questions to avoid"},"filter":{"anyOf":[{"$ref":"#/components/schemas/FilterCriteria"},{"items":{"$ref":"#/components/schemas/FilterCriteria"},"type":"array"},{"type":"null"}],"title":"Filter","description":"Optional filter criteria to apply after question generation"},"questions_per_seed":{"type":"integer","title":"Questions Per Seed","description":"Number of questions to generate per seed","default":1},"include_default_filter":{"type":"boolean","title":"Include Default Filter","description":"Whether to include the default filter for generated questions","default":false},"answer_type":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/BinaryAnswerType"},{"$ref":"#/components/schemas/MultipleChoiceAnswerType"},{"$ref":"#/components/schemas/ContinuousAnswerType"},{"$ref":"#/components/schemas/FreeResponseAnswerType"}],"discriminator":{"propertyName":"answer_type","mapping":{"BINARY":"#/components/schemas/BinaryAnswerType","CONTINUOUS":"#/components/schemas/ContinuousAnswerType","FREE_RESPONSE":"#/components/schemas/FreeResponseAnswerType","MULTIPLE_CHOICE":"#/components/schemas/MultipleChoiceAnswerType"}}},{"type":"null"}],"title":"Answer Type","description":"The type of answer expected for generated questions"}},"type":"object","title":"QuestionGenerator"},"QuestionPipeline":{"properties":{"config_type":{"type":"string","const":"QUESTION_PIPELINE","title":"Config Type","description":"Type of transform configuration","default":"QUESTION_PIPELINE"},"seed_generator":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/BigQuerySeedGenerator"},{"$ref":"#/components/schemas/NewsSeedGenerator"},{"$ref":"#/components/schemas/GdeltSeedGenerator"},{"$ref":"#/components/schemas/FileSetSeedGenerator"},{"$ref":"#/components/schemas/FileSetQuerySeedGenerator"},{"$ref":"#/components/schemas/MockTransformConfig"}],"discriminator":{"propertyName":"config_type","mapping":{"BIGQUERY_SEED_GENERATOR":"#/components/schemas/BigQuerySeedGenerator","FILESET_QUERY_SEED_GENERATOR":"#/components/schemas/FileSetQuerySeedGenerator","FILESET_SEED_GENERATOR":"#/components/schemas/FileSetSeedGenerator","GDELT_SEED_GENERATOR":"#/components/schemas/GdeltSeedGenerator","MOCK":"#/components/schemas/MockTransformConfig","NEWS_SEED_GENERATOR":"#/components/schemas/NewsSeedGenerator"}}},{"type":"null"}],"title":"Seed Generator","description":"Configuration for seed generation"},"question_generator":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/QuestionGenerator"},{"$ref":"#/components/schemas/ForwardLookingQuestionGenerator"},{"$ref":"#/components/schemas/QuestionAndLabelGenerator"},{"$ref":"#/components/schemas/TemplateQuestionGenerator"},{"$ref":"#/components/schemas/MockTransformConfig"}],"discriminator":{"propertyName":"config_type","mapping":{"FORWARD_LOOKING_QUESTION_GENERATOR":"#/components/schemas/ForwardLookingQuestionGenerator","MOCK":"#/components/schemas/MockTransformConfig","QUESTION_AND_LABEL_GENERATOR":"#/components/schemas/QuestionAndLabelGenerator","QUESTION_GENERATOR":"#/components/schemas/QuestionGenerator","TEMPLATE_QUESTION_GENERATOR":"#/components/schemas/TemplateQuestionGenerator"}}},{"type":"null"}],"title":"Question Generator","description":"Configuration for question generation"},"labeler":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/WebSearchLabeler"},{"$ref":"#/components/schemas/FileSetRAGLabeler"},{"$ref":"#/components/schemas/MockTransformConfig"}],"discriminator":{"propertyName":"config_type","mapping":{"FILESET_RAG_LABELER":"#/components/schemas/FileSetRAGLabeler","MOCK":"#/components/schemas/MockTransformConfig","WEB_SEARCH_LABELER":"#/components/schemas/WebSearchLabeler"}}},{"type":"null"}],"title":"Labeler","description":"Configuration for labeling. Not needed when using QuestionAndLabelGenerator."},"context_generators":{"anyOf":[{"items":{"anyOf":[{"$ref":"#/components/schemas/NewsContextGenerator"},{"$ref":"#/components/schemas/FileSetContextGenerator"},{"$ref":"#/components/schemas/MockTransformConfig"}]},"type":"array"},{"type":"null"}],"title":"Context Generators","description":"Optional list of context generators to run before rendering"},"renderer":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/QuestionRenderer"},{"$ref":"#/components/schemas/MockTransformConfig"}],"discriminator":{"propertyName":"config_type","mapping":{"MOCK":"#/components/schemas/MockTransformConfig","QUESTION_RENDERER":"#/components/schemas/QuestionRenderer"}}},{"type":"null"}],"title":"Renderer","description":"Optional configuration for rendering the final prompt"},"rollout_generator":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/RolloutGenerator"},{"$ref":"#/components/schemas/MockTransformConfig"}],"discriminator":{"propertyName":"config_type","mapping":{"MOCK":"#/components/schemas/MockTransformConfig","ROLLOUT_GENERATOR":"#/components/schemas/RolloutGenerator"}}},{"type":"null"}],"title":"Rollout Generator","description":"Optional configuration for generating rollouts from multiple models"},"scorer":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/RolloutScorer"},{"$ref":"#/components/schemas/MockTransformConfig"}],"discriminator":{"propertyName":"config_type","mapping":{"MOCK":"#/components/schemas/MockTransformConfig","ROLLOUT_SCORER":"#/components/schemas/RolloutScorer"}}},{"type":"null"}],"title":"Scorer","description":"Optional configuration for scoring rollouts against ground truth"}},"type":"object","title":"QuestionPipeline"},"BigQuerySeedGenerator":{"properties":{"config_type":{"type":"string","const":"BIGQUERY_SEED_GENERATOR","title":"Config Type","description":"Type of transform configuration","default":"BIGQUERY_SEED_GENERATOR"},"query":{"type":"string","title":"Query","description":"BigQuery SQL query to execute"},"seed_text_column":{"type":"string","title":"Seed Text Column","description":"Column name for Seed.seed_text","default":"text"},"date_column":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Date Column","description":"Column name for Seed.seed_creation_date"},"max_rows":{"type":"integer","title":"Max Rows","description":"Total rows to fetch across all batches","default":10000}},"type":"object","required":["query"],"title":"BigQuerySeedGenerator"},"MockTransformConfig":{"properties":{"config_type":{"type":"string","const":"MOCK","title":"Config Type","default":"MOCK"},"num_seeds":{"type":"integer","title":"Num Seeds","description":"Number of mock seeds to generate (0 = not a seed generator)","default":0},"seed_text_template":{"type":"string","title":"Seed Text Template","description":"Template for seed text","default":"Mock seed content {index}"},"delay_seconds":{"type":"number","title":"Delay Seconds","description":"Simulated processing time per item","default":0.1},"delay_variance":{"type":"number","title":"Delay Variance","description":"Random variance (+/- seconds)","default":0.05},"error_rate":{"type":"number","maximum":1,"minimum":0,"title":"Error Rate","description":"Probability of error (0.0-1.0)","default":0.01},"error_message":{"type":"string","title":"Error Message","description":"Error message when error_rate triggers","default":"Mock error"},"expansion_factor":{"type":"integer","minimum":1,"title":"Expansion Factor","description":"Outputs per input (1:N expansion)","default":1},"filter_rate":{"type":"number","maximum":1,"minimum":0,"title":"Filter Rate","description":"Probability of marking invalid","default":0},"add_question":{"type":"boolean","title":"Add Question","description":"Add a mock question to samples","default":false},"question_template":{"type":"string","title":"Question Template","description":"Template for question text","default":"Mock question {index}?"},"metadata_additions":{"additionalProperties":true,"type":"object","title":"Metadata Additions","description":"Added to sample.meta"},"estimated_input_tokens":{"type":"integer","title":"Estimated Input Tokens","description":"Estimated input tokens per call","default":0},"estimated_output_tokens":{"type":"integer","title":"Estimated Output Tokens","description":"Estimated output tokens per call","default":0},"estimated_model_name":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Estimated Model Name","description":"Model name for cost estimation"},"simulated_cost_per_call":{"type":"number","title":"Simulated Cost Per Call","description":"Simulated cost per call in dollars (records usage event when > 0)","default":0},"random_seed":{"anyOf":[{"type":"integer"},{"type":"null"}],"title":"Random Seed","description":"Seed for deterministic behavior"}},"type":"object","title":"MockTransformConfig"},"TemplateQuestionGenerator":{"properties":{"config_type":{"type":"string","const":"TEMPLATE_QUESTION_GENERATOR","title":"Config Type","description":"Type of transform configuration","default":"TEMPLATE_QUESTION_GENERATOR"},"question_template":{"type":"string","title":"Question Template","description":"Template string with {field} placeholders. Supports {seed_text}, {seed}, and any key in {meta} (e.g. {meta.label})."}},"type":"object","required":["question_template"],"title":"TemplateQuestionGenerator"},"WebSearchLabeler":{"properties":{"config_type":{"type":"string","const":"WEB_SEARCH_LABELER","title":"Config Type","description":"Type of transform configuration","default":"WEB_SEARCH_LABELER"},"confidence_threshold":{"type":"number","title":"Confidence Threshold","description":"Minimum confidence threshold for including questions","default":0.9},"answer_type":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/BinaryAnswerType"},{"$ref":"#/components/schemas/MultipleChoiceAnswerType"},{"$ref":"#/components/schemas/ContinuousAnswerType"},{"$ref":"#/components/schemas/FreeResponseAnswerType"}],"discriminator":{"propertyName":"answer_type","mapping":{"BINARY":"#/components/schemas/BinaryAnswerType","CONTINUOUS":"#/components/schemas/ContinuousAnswerType","FREE_RESPONSE":"#/components/schemas/FreeResponseAnswerType","MULTIPLE_CHOICE":"#/components/schemas/MultipleChoiceAnswerType"}}},{"type":"null"}],"title":"Answer Type","description":"The type of answer expected, used to guide the labeler"},"resolve_redirects":{"type":"boolean","title":"Resolve Redirects","description":"Resolve redirect URLs to actual destinations","default":false}},"type":"object","title":"WebSearchLabeler"},"FileSetRAGLabeler":{"properties":{"config_type":{"type":"string","const":"FILESET_RAG_LABELER","title":"Config Type","description":"Type of transform configuration","default":"FILESET_RAG_LABELER"},"file_set_id":{"type":"string","title":"File Set Id","description":"FileSet ID to query for labeling"},"metadata_filter_keys":{"items":{"type":"string"},"type":"array","title":"Metadata Filter Keys","description":"Keys from seed's file_metadata for dynamic filtering (e.g., ['ticker'])"},"metadata_filter":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Metadata Filter","description":"Static AIP-160 metadata filter (combined with dynamic via AND)"},"system_instruction":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"System Instruction","description":"Optional system instruction for the Gemini RAG query"},"confidence_threshold":{"type":"number","title":"Confidence Threshold","description":"Minimum confidence threshold for including questions","default":0.9},"answer_type":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/BinaryAnswerType"},{"$ref":"#/components/schemas/MultipleChoiceAnswerType"},{"$ref":"#/components/schemas/ContinuousAnswerType"},{"$ref":"#/components/schemas/FreeResponseAnswerType"}],"discriminator":{"propertyName":"answer_type","mapping":{"BINARY":"#/components/schemas/BinaryAnswerType","CONTINUOUS":"#/components/schemas/ContinuousAnswerType","FREE_RESPONSE":"#/components/schemas/FreeResponseAnswerType","MULTIPLE_CHOICE":"#/components/schemas/MultipleChoiceAnswerType"}}},{"type":"null"}],"title":"Answer Type","description":"The type of answer expected, used to guide the labeler"},"temporal_constraint":{"anyOf":[{"$ref":"#/components/schemas/TemporalConstraint"},{"type":"null"}],"description":"Filter documents by date relative to seed date. AFTER (>) for resolution docs, BEFORE (<=) for historical only."},"date_metadata_key":{"type":"string","title":"Date Metadata Key","description":"Gemini metadata key storing unix timestamp for temporal filtering","default":"file_date"},"model":{"type":"string","title":"Model","description":"Gemini model for RAG query","default":"gemini-3-flash-preview"}},"type":"object","required":["file_set_id"],"title":"FileSetRAGLabeler","description":"Configuration for FileSet RAG Labeler transform."},"TemporalConstraint":{"type":"string","enum":["BEFORE","AFTER"],"title":"TemporalConstraint","description":"Temporal filtering direction relative to the seed document's date.\n\nUses the `file_date` metadata key (unix timestamp int) stored on each\nGemini document by fileset_file_processor.\n\nBEFORE: file_date <= seed_timestamp  (context: no lookahead bias)\nAFTER:  file_date >  seed_timestamp  (labels: find resolutions)"},"NewsContextGenerator":{"properties":{"config_type":{"type":"string","const":"NEWS_CONTEXT_GENERATOR","title":"Config Type","description":"Type of transform configuration","default":"NEWS_CONTEXT_GENERATOR"},"num_search_queries":{"type":"integer","title":"Num Search Queries","description":"Number of search queries to generate per question","default":5},"articles_per_query":{"type":"integer","title":"Articles Per Query","description":"Number of news articles to return per search query","default":3},"num_articles":{"type":"integer","title":"Num Articles","description":"Maximum number of news articles to include in final output","default":10},"relevance_threshold":{"type":"integer","title":"Relevance Threshold","description":"Minimum relevance rating (1-6 scale) to include article","default":2},"min_articles":{"type":"integer","title":"Min Articles","description":"Minimum number of articles to ensure","default":6},"time_delta_days":{"type":"integer","title":"Time Delta Days","description":"Number of days to look back for news articles","default":30},"enable_relevance_ranking":{"type":"boolean","title":"Enable Relevance Ranking","description":"Whether to perform LLM-based relevance ranking","default":true}},"type":"object","title":"NewsContextGenerator"},"FileSetContextGenerator":{"properties":{"config_type":{"type":"string","const":"FILESET_CONTEXT_GENERATOR","title":"Config Type","description":"Type of transform configuration","default":"FILESET_CONTEXT_GENERATOR"},"file_set_id":{"type":"string","title":"File Set Id","description":"FileSet ID to query for context"},"metadata_filter_keys":{"items":{"type":"string"},"type":"array","title":"Metadata Filter Keys","description":"Keys from seed's file_metadata for dynamic filtering (e.g., ['ticker'])"},"metadata_filter":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Metadata Filter","description":"Static AIP-160 metadata filter (combined with dynamic via AND)"},"system_instruction":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"System Instruction","description":"Optional system instruction for the Gemini model"},"query_template":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Query Template","description":"Template with {question} placeholder; default: raw question text"},"temporal_constraint":{"anyOf":[{"$ref":"#/components/schemas/TemporalConstraint"},{"type":"null"}],"description":"Filter documents by date relative to seed date. BEFORE (<=) for context without lookahead, AFTER (>) for future docs."},"date_metadata_key":{"type":"string","title":"Date Metadata Key","description":"Gemini metadata key storing unix timestamp for temporal filtering","default":"file_date"},"model":{"type":"string","title":"Model","description":"Gemini model to use for file search","default":"gemini-3-flash-preview"}},"type":"object","required":["file_set_id"],"title":"FileSetContextGenerator","description":"Configuration for FileSet Context Generator transform."},"QuestionRenderer":{"properties":{"config_type":{"type":"string","const":"QUESTION_RENDERER","title":"Config Type","description":"Type of transform configuration","default":"QUESTION_RENDERER"},"template":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Template","description":"Custom template for rendering the prompt. If not provided, dynamically builds based on available content. Supports placeholders like {question_text}, {context}, {answer_instructions}."},"answer_type":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/BinaryAnswerType"},{"$ref":"#/components/schemas/MultipleChoiceAnswerType"},{"$ref":"#/components/schemas/ContinuousAnswerType"},{"$ref":"#/components/schemas/FreeResponseAnswerType"}],"discriminator":{"propertyName":"answer_type","mapping":{"BINARY":"#/components/schemas/BinaryAnswerType","CONTINUOUS":"#/components/schemas/ContinuousAnswerType","FREE_RESPONSE":"#/components/schemas/FreeResponseAnswerType","MULTIPLE_CHOICE":"#/components/schemas/MultipleChoiceAnswerType"}}},{"type":"null"}],"title":"Answer Type","description":"The type of answer expected, used to render answer instructions"}},"type":"object","title":"QuestionRenderer"},"RolloutGenerator":{"properties":{"config_type":{"type":"string","const":"ROLLOUT_GENERATOR","title":"Config Type","default":"ROLLOUT_GENERATOR"},"models":{"items":{"$ref":"#/components/schemas/ModelConfig"},"type":"array","title":"Models","description":"Model names or ModelConfig objects"},"prompt_template":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Prompt Template","description":"Prompt template with {column} placeholders. If None, uses sample.prompt"},"input_columns":{"items":{"type":"string"},"type":"array","title":"Input Columns","description":"Columns to substitute into template (from meta)"},"output_schema":{"anyOf":[{},{"type":"null"}],"title":"Output Schema","description":"Pydantic model for structured output"}},"type":"object","required":["models"],"title":"RolloutGenerator"},"ModelConfig":{"properties":{"model_name":{"type":"string","title":"Model Name","default":"meta-llama/llama-3.3-70b-instruct"},"model_source":{"$ref":"#/components/schemas/ModelSourceType","default":"OPEN_ROUTER"},"temperature":{"type":"number","title":"Temperature","default":1},"max_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}],"title":"Max Tokens"},"ip_address":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Ip Address"},"lora_base_model_name":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Lora Base Model Name"},"lora_repo_path":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Lora Repo Path"},"lora_checkpoint_path":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Lora Checkpoint Path"},"runpod_endpoint_id":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Runpod Endpoint Id"},"is_lightningrod_model":{"anyOf":[{"type":"boolean"},{"type":"null"}],"title":"Is Lightningrod Model"},"openrouter_provider":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}],"title":"Openrouter Provider"},"reasoning_effort":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Reasoning Effort"},"is_reasoning_model":{"anyOf":[{"type":"boolean"},{"type":"null"}],"title":"Is Reasoning Model"},"disable_reasoning":{"type":"boolean","title":"Disable Reasoning","default":false},"use_pipeline_key":{"type":"boolean","title":"Use Pipeline Key","default":false}},"type":"object","title":"ModelConfig"},"ModelSourceType":{"type":"string","enum":["OPENAI","VLLM","DIRECT_COMPLETIONS","OPEN_ROUTER","HUGGINGFACE_ENDPOINTS","DEEPSEEK","DEEPINFRA","RUNPOD","SGLANG","TINKER","MODAL_LORA_VLLM"],"title":"ModelSourceType"},"RolloutScorer":{"properties":{"config_type":{"type":"string","const":"ROLLOUT_SCORER","title":"Config Type","default":"ROLLOUT_SCORER"},"answer_type":{"oneOf":[{"$ref":"#/components/schemas/BinaryAnswerType"},{"$ref":"#/components/schemas/MultipleChoiceAnswerType"},{"$ref":"#/components/schemas/ContinuousAnswerType"},{"$ref":"#/components/schemas/FreeResponseAnswerType"}],"title":"Answer Type","description":"the answer type of the question","discriminator":{"propertyName":"answer_type","mapping":{"BINARY":"#/components/schemas/BinaryAnswerType","CONTINUOUS":"#/components/schemas/ContinuousAnswerType","FREE_RESPONSE":"#/components/schemas/FreeResponseAnswerType","MULTIPLE_CHOICE":"#/components/schemas/MultipleChoiceAnswerType"}}},"multiple_choice_options":{"anyOf":[{"additionalProperties":{"type":"string"},"type":"object"},{"type":"null"}],"title":"Multiple Choice Options"},"is_mutually_exclusive":{"type":"boolean","title":"Is Mutually Exclusive","default":true}},"type":"object","required":["answer_type"],"title":"RolloutScorer"},"TransformJob":{"properties":{"id":{"type":"string","title":"Id"},"organization_id":{"type":"string","title":"Organization Id"},"configuration_id":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Configuration Id"},"name":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Name"},"status":{"$ref":"#/components/schemas/TransformJobStatus"},"error_message":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Error Message"},"warning_message":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Warning Message"},"modal_function_call_id":{"type":"string","title":"Modal Function Call Id"},"modal_app_id":{"type":"string","title":"Modal App Id"},"transform_config":{"type":"string","title":"Transform Config"},"input_dataset_id":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Input Dataset Id"},"output_dataset_id":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Output Dataset Id"},"usage":{"anyOf":[{"$ref":"#/components/schemas/JobUsage"},{"type":"null"}]},"estimated_cost_dollars":{"anyOf":[{"type":"number"},{"type":"null"}],"title":"Estimated Cost Dollars"},"created_at":{"type":"string","format":"date-time","title":"Created At"},"updated_at":{"type":"string","format":"date-time","title":"Updated At"}},"type":"object","required":["id","organization_id","status","modal_function_call_id","modal_app_id","transform_config","input_dataset_id","output_dataset_id","created_at","updated_at"],"title":"TransformJob"},"TransformJobStatus":{"type":"string","enum":["RUNNING","COMPLETED","FAILED","CANCELLED"],"title":"TransformJobStatus"},"JobUsage":{"properties":{"total":{"anyOf":[{"$ref":"#/components/schemas/UsageSummary"},{"type":"null"}]},"by_step":{"anyOf":[{"additionalProperties":{"$ref":"#/components/schemas/UsageSummary"},"type":"object"},{"type":"null"}],"title":"By Step"},"max_cost_dollars":{"anyOf":[{"type":"number"},{"type":"null"}],"title":"Max Cost Dollars"},"current_cost_dollars":{"anyOf":[{"type":"number"},{"type":"null"}],"title":"Current Cost Dollars"},"estimated_cost_dollars":{"anyOf":[{"type":"number"},{"type":"null"}],"title":"Estimated Cost Dollars"},"pipeline_summary":{"anyOf":[{"items":{"$ref":"#/components/schemas/PipelineStepSummary"},"type":"array"},{"type":"null"}],"title":"Pipeline Summary"}},"type":"object","title":"JobUsage","description":"Aggregated usage statistics for a transform job with step breakdown."},"UsageSummary":{"properties":{"events":{"additionalProperties":{"$ref":"#/components/schemas/EventUsageSummary"},"type":"object","title":"Events","default":{}},"llm_by_model":{"additionalProperties":{"$ref":"#/components/schemas/LLMModelUsageSummary"},"type":"object","title":"Llm By Model","default":{}},"total_cost":{"type":"number","title":"Total Cost","default":0}},"type":"object","title":"UsageSummary","description":"Flexible usage statistics by event type and LLM model."},"EventUsageSummary":{"properties":{"count":{"type":"integer","title":"Count","default":0},"amount":{"type":"number","title":"Amount","default":0},"cost":{"type":"number","title":"Cost","default":0},"cached_count":{"type":"integer","title":"Cached Count","default":0}},"type":"object","title":"EventUsageSummary","description":"Usage for a single event type."},"LLMModelUsageSummary":{"properties":{"count":{"type":"integer","title":"Count","default":0},"input_tokens":{"type":"integer","title":"Input Tokens","default":0},"output_tokens":{"type":"integer","title":"Output Tokens","default":0},"cost":{"type":"number","title":"Cost","default":0},"cached_count":{"type":"integer","title":"Cached Count","default":0}},"type":"object","title":"LLMModelUsageSummary","description":"Usage for a single LLM model."},"PipelineStepSummary":{"properties":{"step_index":{"type":"integer","title":"Step Index"},"transform_name":{"type":"string","title":"Transform Name"},"input_count":{"type":"integer","title":"Input Count","default":0},"output_count":{"type":"integer","title":"Output Count","default":0},"rejected_count":{"type":"integer","title":"Rejected Count","default":0},"error_count":{"type":"integer","title":"Error Count","default":0},"rejection_reasons":{"additionalProperties":{"type":"integer"},"type":"object","title":"Rejection Reasons","default":{}}},"type":"object","required":["step_index","transform_name"],"title":"PipelineStepSummary","description":"Summary of a single pipeline step including rejection reasons."},"HTTPValidationError":{"properties":{"detail":{"items":{"$ref":"#/components/schemas/ValidationError"},"type":"array","title":"Detail"}},"type":"object","title":"HTTPValidationError"},"ValidationError":{"properties":{"loc":{"items":{"anyOf":[{"type":"string"},{"type":"integer"}]},"type":"array","title":"Location"},"msg":{"type":"string","title":"Message"},"type":{"type":"string","title":"Error Type"},"input":{"title":"Input"},"ctx":{"type":"object","title":"Context"}},"type":"object","required":["loc","msg","type"],"title":"ValidationError"}}},"paths":{"/transform-jobs":{"post":{"tags":["Transform Jobs"],"summary":"Create Transform Job","description":"Create a new transform job","operationId":"create_transform_job_transform_jobs_post","requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateTransformJobRequest"}}}},"responses":{"201":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TransformJob"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}}}}
```

## Cost Estimation

> Estimate maximum cost for a transform job before execution

```json
{"openapi":"3.1.0","info":{"title":"LightningRod API","version":"1.0.0"},"servers":[{"url":"/api/public/v1"}],"security":[{"HTTPBearer":[]}],"components":{"securitySchemes":{"HTTPBearer":{"type":"http","scheme":"bearer"}},"schemas":{"EstimateCostRequest":{"properties":{"config":{"$ref":"#/components/schemas/CreateTransformConfig"},"max_questions":{"anyOf":[{"type":"integer"},{"type":"null"}],"title":"Max Questions"}},"type":"object","required":["config"],"title":"EstimateCostRequest"},"CreateTransformConfig":{"oneOf":[{"$ref":"#/components/schemas/ForwardLookingQuestionGenerator"},{"$ref":"#/components/schemas/FileSetQuerySeedGenerator"},{"$ref":"#/components/schemas/FileSetSeedGenerator"},{"$ref":"#/components/schemas/GdeltSeedGenerator"},{"$ref":"#/components/schemas/NewsSeedGenerator"},{"$ref":"#/components/schemas/QuestionAndLabelGenerator"},{"$ref":"#/components/schemas/QuestionGenerator"},{"$ref":"#/components/schemas/QuestionPipeline"},{"$ref":"#/components/schemas/QuestionRenderer"},{"$ref":"#/components/schemas/WebSearchLabeler"}],"discriminator":{"propertyName":"config_type","mapping":{"FILESET_QUERY_SEED_GENERATOR":"#/components/schemas/FileSetQuerySeedGenerator","FILESET_SEED_GENERATOR":"#/components/schemas/FileSetSeedGenerator","FORWARD_LOOKING_QUESTION_GENERATOR":"#/components/schemas/ForwardLookingQuestionGenerator","GDELT_SEED_GENERATOR":"#/components/schemas/GdeltSeedGenerator","NEWS_SEED_GENERATOR":"#/components/schemas/NewsSeedGenerator","QUESTION_AND_LABEL_GENERATOR":"#/components/schemas/QuestionAndLabelGenerator","QUESTION_GENERATOR":"#/components/schemas/QuestionGenerator","QUESTION_PIPELINE":"#/components/schemas/QuestionPipeline","QUESTION_RENDERER":"#/components/schemas/QuestionRenderer","WEB_SEARCH_LABELER":"#/components/schemas/WebSearchLabeler"}}},"ForwardLookingQuestionGenerator":{"properties":{"config_type":{"type":"string","const":"FORWARD_LOOKING_QUESTION_GENERATOR","title":"Config Type","description":"Type of transform configuration","default":"FORWARD_LOOKING_QUESTION_GENERATOR"},"instructions":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Instructions","description":"Instructions for question generation. If not provided, uses sensible defaults."},"bad_examples":{"items":{"type":"string"},"type":"array","title":"Bad Examples","description":"Examples of questions to avoid"},"filter":{"anyOf":[{"$ref":"#/components/schemas/FilterCriteria"},{"items":{"$ref":"#/components/schemas/FilterCriteria"},"type":"array"},{"type":"null"}],"title":"Filter","description":"Optional filter criteria to apply after question generation"},"questions_per_seed":{"type":"integer","title":"Questions Per Seed","description":"Number of questions to generate per seed","default":1},"include_default_filter":{"type":"boolean","title":"Include Default Filter","description":"Whether to include the default filter for generated questions","default":false},"answer_type":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/BinaryAnswerType"},{"$ref":"#/components/schemas/MultipleChoiceAnswerType"},{"$ref":"#/components/schemas/ContinuousAnswerType"},{"$ref":"#/components/schemas/FreeResponseAnswerType"}],"discriminator":{"propertyName":"answer_type","mapping":{"BINARY":"#/components/schemas/BinaryAnswerType","CONTINUOUS":"#/components/schemas/ContinuousAnswerType","FREE_RESPONSE":"#/components/schemas/FreeResponseAnswerType","MULTIPLE_CHOICE":"#/components/schemas/MultipleChoiceAnswerType"}}},{"type":"null"}],"title":"Answer Type","description":"The type of answer expected for generated questions"}},"type":"object","title":"ForwardLookingQuestionGenerator"},"FilterCriteria":{"properties":{"rubric":{"type":"string","title":"Rubric","description":"Scoring rubric/prompt for evaluating content"},"min_score":{"type":"number","maximum":1,"minimum":0,"title":"Min Score","description":"Minimum score threshold","default":0.5},"model_name":{"type":"string","title":"Model Name","description":"Name of the model (in openrouter) to use for scoring","default":"google/gemini-3-flash-preview"}},"type":"object","required":["rubric"],"title":"FilterCriteria","description":"Reusable filter criteria for LLM-based content scoring and filtering."},"BinaryAnswerType":{"properties":{"answer_type":{"type":"string","const":"BINARY","title":"Answer Type","default":"BINARY"},"answer_format_instruction":{"type":"string","title":"Answer Format Instruction","description":"Instructions describing how the answer should be formatted and given.","default":"This is a binary yes/no question. You are estimating the probability that the answer is 'Yes'. Provide your confidence as a value between 0 (definitely No) and 1 (definitely Yes). Provide your probability estimate for Yes as a decimal between 0 and 1. Provide your answer between <answer></answer> tags. Example: <answer>0.75</answer>"},"labeler_instruction":{"type":"string","title":"Labeler Instruction","description":"Instructions for the labeler.","default":"The answer should be ONLY '1', '0', or 'Undetermined'. '1' means yes, '0' means no, and 'Undetermined' means the answer is not clear. Do not include any other text or explanation."},"question_generation_instruction":{"type":"string","title":"Question Generation Instruction","description":"Instructions for generating questions of this type.","default":"Generate questions that can be answered with Yes or No. Frame the question so it has exactly two possible outcomes: Yes or No. The question should start with words like 'Will', 'Is', 'Does', 'Has', 'Can', etc. A clear and unambiguous yes/no question, based on the provided seed_text."},"reward_function_type":{"anyOf":[{"$ref":"#/components/schemas/RewardFunctionType"},{"type":"null"}],"default":"binary_brier"}},"type":"object","title":"BinaryAnswerType"},"RewardFunctionType":{"type":"string","enum":["binary_brier","binary_weighted_brier","binary_log_score","multi_choice_log_score","continuous_log_score","continuous_value_only_log_score","continuous_value_only_linear_score"],"title":"RewardFunctionType"},"MultipleChoiceAnswerType":{"properties":{"answer_type":{"type":"string","const":"MULTIPLE_CHOICE","title":"Answer Type","default":"MULTIPLE_CHOICE"},"answer_format_instruction":{"type":"string","title":"Answer Format Instruction","description":"Instructions describing how the answer should be formatted and given.","default":"This is a multiple choice question with answer options. The list of options will be provided in the question. You are estimating the probability for each option being the correct answer. Provide your confidence for each option as a value between 0 and 1, where the probabilities must sum to 1. Provide your answer as a JSON dictionary between <answer></answer> tags, with keys like option_0, option_1, etc. corresponding to each option in order. Example: <answer>{\"option_0\": 0.3, \"option_1\": 0.4, \"option_2\": 0.2, \"option_3\": 0.1}</answer>"},"labeler_instruction":{"type":"string","title":"Labeler Instruction","description":"Instructions for the labeler.","default":"The answer should be ONLY one of the following options: 'A', 'B', 'C', or 'D', or 'Undetermined'. Do not include any other text or explanation."},"question_generation_instruction":{"type":"string","title":"Question Generation Instruction","description":"Instructions for generating questions of this type.","default":"Generate questions with multiple choice options (up to 10). Format the options on separate lines after the question:\nA) First option\nB) Second option\nC) Third option\n... and so on\n\nEach option should be distinct and mutually exclusive. A clear and unambiguous multiple-choice question, based on the provided seed_text. Include the options in the question text."},"reward_function_type":{"anyOf":[{"$ref":"#/components/schemas/RewardFunctionType"},{"type":"null"}],"default":"multi_choice_log_score"}},"type":"object","title":"MultipleChoiceAnswerType"},"ContinuousAnswerType":{"properties":{"answer_type":{"type":"string","const":"CONTINUOUS","title":"Answer Type","default":"CONTINUOUS"},"answer_format_instruction":{"type":"string","title":"Answer Format Instruction","description":"Instructions describing how the answer should be formatted and given.","default":"This question expects a numeric value as the answer. Provide your best estimate as a single number. Include units if specified in the question. Provide your answer between <answer></answer> tags. Example: <answer>42.5</answer>"},"labeler_instruction":{"type":"string","title":"Labeler Instruction","description":"Instructions for the labeler.","default":"The answer should be ONLY a single exact numeric value, not a range. For example: '42.5' or '1000', not '40-45' or 'between 900 and 1100', or 'Undetermined'. Do not include any other text or explanation."},"question_generation_instruction":{"type":"string","title":"Question Generation Instruction","description":"Instructions for generating questions of this type.","default":"Generate questions that expect a numeric value as the answer. Specify the units if applicable (e.g., dollars, percent, count). A clear and unambiguous question, based on the provided seed_text, that expects a numeric value as the answer."},"reward_function_type":{"anyOf":[{"$ref":"#/components/schemas/RewardFunctionType"},{"type":"null"}],"description":"Reward function type for scoring rollouts. None for answer types that don't support scoring."}},"type":"object","title":"ContinuousAnswerType"},"FreeResponseAnswerType":{"properties":{"answer_type":{"type":"string","const":"FREE_RESPONSE","title":"Answer Type","default":"FREE_RESPONSE"},"answer_format_instruction":{"type":"string","title":"Answer Format Instruction","description":"Instructions describing how the answer should be formatted and given.","default":"This question expects a free-form text response. Provide an answer that directly addresses what the question is asking. Provide your answer between <answer></answer> tags. Example: <answer>The company announced a new product line.</answer>"},"labeler_instruction":{"type":"string","title":"Labeler Instruction","description":"Instructions for the labeler.","default":"Respond with the correct answer as a text description."},"question_generation_instruction":{"type":"string","title":"Question Generation Instruction","description":"Instructions for generating questions of this type.","default":"Generate questions that expect a free-form text response. A clear and unambiguous question, based on the provided seed_text, that expects a free-form text response."},"reward_function_type":{"anyOf":[{"$ref":"#/components/schemas/RewardFunctionType"},{"type":"null"}],"description":"Reward function type for scoring rollouts. None for answer types that don't support scoring."}},"type":"object","title":"FreeResponseAnswerType"},"FileSetQuerySeedGenerator":{"properties":{"config_type":{"type":"string","const":"FILESET_QUERY_SEED_GENERATOR","title":"Config Type","description":"Type of transform configuration","default":"FILESET_QUERY_SEED_GENERATOR"},"file_set_id":{"type":"string","title":"File Set Id","description":"FileSet ID to query"},"prompts":{"items":{"type":"string"},"type":"array","title":"Prompts","description":"List of queries to run against the fileset"},"metadata_filters":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}],"title":"Metadata Filters","description":"Optional list of AIP-160 metadata filters to select which documents to process. Documents matching ANY filter will be included. (e.g., [\"ticker='AAL'\", \"ticker='MSFT'\"])"},"system_instruction":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"System Instruction","description":"Optional system instruction for the Gemini model"}},"type":"object","required":["file_set_id","prompts"],"title":"FileSetQuerySeedGenerator","description":"Configuration for FileSet Query Seed Generator transform."},"FileSetSeedGenerator":{"properties":{"config_type":{"type":"string","const":"FILESET_SEED_GENERATOR","title":"Config Type","description":"Type of transform configuration","default":"FILESET_SEED_GENERATOR"},"file_set_id":{"type":"string","title":"File Set Id","description":"FileSet ID to read files from"},"chunk_size":{"type":"integer","minimum":100,"title":"Chunk Size","description":"Number of characters per chunk","default":4000},"chunk_overlap":{"type":"integer","minimum":0,"title":"Chunk Overlap","description":"Number of overlapping characters between consecutive chunks","default":200},"metadata_filters":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}],"title":"Metadata Filters","description":"Optional list of metadata filters to select which files to process. Files matching ANY filter will be included. (e.g., [\"ticker='AAL'\", \"ticker='MSFT'\"])"}},"type":"object","required":["file_set_id"],"title":"FileSetSeedGenerator","description":"Configuration for FileSet Seed Generator transform."},"GdeltSeedGenerator":{"properties":{"config_type":{"type":"string","const":"GDELT_SEED_GENERATOR","title":"Config Type","description":"Type of transform configuration","default":"GDELT_SEED_GENERATOR"},"start_date":{"type":"string","format":"date-time","title":"Start Date","description":"Start date for seed search"},"end_date":{"type":"string","format":"date-time","title":"End Date","description":"End date for seed search"},"interval_duration_days":{"type":"integer","title":"Interval Duration Days","description":"Duration of each interval in days","default":7},"articles_per_interval":{"type":"integer","title":"Articles Per Interval","description":"Number of articles to fetch per interval from BigQuery","default":1000}},"type":"object","required":["start_date","end_date"],"title":"GdeltSeedGenerator"},"NewsSeedGenerator":{"properties":{"config_type":{"type":"string","const":"NEWS_SEED_GENERATOR","title":"Config Type","description":"Type of transform configuration","default":"NEWS_SEED_GENERATOR"},"start_date":{"type":"string","format":"date-time","title":"Start Date","description":"Start date for seed search"},"end_date":{"type":"string","format":"date-time","title":"End Date","description":"End date for seed search"},"interval_duration_days":{"type":"integer","title":"Interval Duration Days","description":"Duration of each interval in days","default":7},"search_query":{"anyOf":[{"type":"string"},{"items":{"type":"string"},"type":"array"}],"title":"Search Query","description":"Search query for news articles. If multiple queries are provided, a separate search will be done for each query."},"articles_per_search":{"type":"integer","title":"Articles Per Search","description":"Number of articles to fetch per search (max 100). Each query/domain combination is a separate search.","default":10},"filter_criteria":{"anyOf":[{"$ref":"#/components/schemas/FilterCriteria"},{"items":{"$ref":"#/components/schemas/FilterCriteria"},"type":"array"},{"type":"null"}],"title":"Filter Criteria","description":"Optional criteria for filtering news snippets before scraping"},"source_domain":{"anyOf":[{"type":"string"},{"items":{"type":"string"},"type":"array"},{"type":"null"}],"title":"Source Domain","description":"Optional URL source of the news articles, e.g. 'https://reuters.com/business', if multiple sources are provided, multiple searchers will be done for each interval"}},"type":"object","required":["start_date","end_date","search_query"],"title":"NewsSeedGenerator"},"QuestionAndLabelGenerator":{"properties":{"config_type":{"type":"string","const":"QUESTION_AND_LABEL_GENERATOR","title":"Config Type","description":"Type of transform configuration","default":"QUESTION_AND_LABEL_GENERATOR"},"instructions":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Instructions","description":"Instructions for question generation. If not provided, uses sensible defaults."},"bad_examples":{"items":{"type":"string"},"type":"array","title":"Bad Examples","description":"Examples of questions to avoid"},"filter":{"anyOf":[{"$ref":"#/components/schemas/FilterCriteria"},{"items":{"$ref":"#/components/schemas/FilterCriteria"},"type":"array"},{"type":"null"}],"title":"Filter","description":"Optional filter criteria to apply after question generation"},"questions_per_seed":{"type":"integer","title":"Questions Per Seed","description":"Number of questions to generate per seed","default":1},"include_default_filter":{"type":"boolean","title":"Include Default Filter","description":"Whether to include the default filter for generated questions","default":false},"answer_type":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/BinaryAnswerType"},{"$ref":"#/components/schemas/MultipleChoiceAnswerType"},{"$ref":"#/components/schemas/ContinuousAnswerType"},{"$ref":"#/components/schemas/FreeResponseAnswerType"}],"discriminator":{"propertyName":"answer_type","mapping":{"BINARY":"#/components/schemas/BinaryAnswerType","CONTINUOUS":"#/components/schemas/ContinuousAnswerType","FREE_RESPONSE":"#/components/schemas/FreeResponseAnswerType","MULTIPLE_CHOICE":"#/components/schemas/MultipleChoiceAnswerType"}}},{"type":"null"}],"title":"Answer Type","description":"The type of answer expected for generated questions"}},"type":"object","title":"QuestionAndLabelGenerator"},"QuestionGenerator":{"properties":{"config_type":{"type":"string","const":"QUESTION_GENERATOR","title":"Config Type","description":"Type of transform configuration","default":"QUESTION_GENERATOR"},"instructions":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Instructions","description":"Instructions for question generation. If not provided, uses sensible defaults."},"bad_examples":{"items":{"type":"string"},"type":"array","title":"Bad Examples","description":"Examples of questions to avoid"},"filter":{"anyOf":[{"$ref":"#/components/schemas/FilterCriteria"},{"items":{"$ref":"#/components/schemas/FilterCriteria"},"type":"array"},{"type":"null"}],"title":"Filter","description":"Optional filter criteria to apply after question generation"},"questions_per_seed":{"type":"integer","title":"Questions Per Seed","description":"Number of questions to generate per seed","default":1},"include_default_filter":{"type":"boolean","title":"Include Default Filter","description":"Whether to include the default filter for generated questions","default":false},"answer_type":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/BinaryAnswerType"},{"$ref":"#/components/schemas/MultipleChoiceAnswerType"},{"$ref":"#/components/schemas/ContinuousAnswerType"},{"$ref":"#/components/schemas/FreeResponseAnswerType"}],"discriminator":{"propertyName":"answer_type","mapping":{"BINARY":"#/components/schemas/BinaryAnswerType","CONTINUOUS":"#/components/schemas/ContinuousAnswerType","FREE_RESPONSE":"#/components/schemas/FreeResponseAnswerType","MULTIPLE_CHOICE":"#/components/schemas/MultipleChoiceAnswerType"}}},{"type":"null"}],"title":"Answer Type","description":"The type of answer expected for generated questions"}},"type":"object","title":"QuestionGenerator"},"QuestionPipeline":{"properties":{"config_type":{"type":"string","const":"QUESTION_PIPELINE","title":"Config Type","description":"Type of transform configuration","default":"QUESTION_PIPELINE"},"seed_generator":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/BigQuerySeedGenerator"},{"$ref":"#/components/schemas/NewsSeedGenerator"},{"$ref":"#/components/schemas/GdeltSeedGenerator"},{"$ref":"#/components/schemas/FileSetSeedGenerator"},{"$ref":"#/components/schemas/FileSetQuerySeedGenerator"},{"$ref":"#/components/schemas/MockTransformConfig"}],"discriminator":{"propertyName":"config_type","mapping":{"BIGQUERY_SEED_GENERATOR":"#/components/schemas/BigQuerySeedGenerator","FILESET_QUERY_SEED_GENERATOR":"#/components/schemas/FileSetQuerySeedGenerator","FILESET_SEED_GENERATOR":"#/components/schemas/FileSetSeedGenerator","GDELT_SEED_GENERATOR":"#/components/schemas/GdeltSeedGenerator","MOCK":"#/components/schemas/MockTransformConfig","NEWS_SEED_GENERATOR":"#/components/schemas/NewsSeedGenerator"}}},{"type":"null"}],"title":"Seed Generator","description":"Configuration for seed generation"},"question_generator":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/QuestionGenerator"},{"$ref":"#/components/schemas/ForwardLookingQuestionGenerator"},{"$ref":"#/components/schemas/QuestionAndLabelGenerator"},{"$ref":"#/components/schemas/TemplateQuestionGenerator"},{"$ref":"#/components/schemas/MockTransformConfig"}],"discriminator":{"propertyName":"config_type","mapping":{"FORWARD_LOOKING_QUESTION_GENERATOR":"#/components/schemas/ForwardLookingQuestionGenerator","MOCK":"#/components/schemas/MockTransformConfig","QUESTION_AND_LABEL_GENERATOR":"#/components/schemas/QuestionAndLabelGenerator","QUESTION_GENERATOR":"#/components/schemas/QuestionGenerator","TEMPLATE_QUESTION_GENERATOR":"#/components/schemas/TemplateQuestionGenerator"}}},{"type":"null"}],"title":"Question Generator","description":"Configuration for question generation"},"labeler":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/WebSearchLabeler"},{"$ref":"#/components/schemas/FileSetRAGLabeler"},{"$ref":"#/components/schemas/MockTransformConfig"}],"discriminator":{"propertyName":"config_type","mapping":{"FILESET_RAG_LABELER":"#/components/schemas/FileSetRAGLabeler","MOCK":"#/components/schemas/MockTransformConfig","WEB_SEARCH_LABELER":"#/components/schemas/WebSearchLabeler"}}},{"type":"null"}],"title":"Labeler","description":"Configuration for labeling. Not needed when using QuestionAndLabelGenerator."},"context_generators":{"anyOf":[{"items":{"anyOf":[{"$ref":"#/components/schemas/NewsContextGenerator"},{"$ref":"#/components/schemas/FileSetContextGenerator"},{"$ref":"#/components/schemas/MockTransformConfig"}]},"type":"array"},{"type":"null"}],"title":"Context Generators","description":"Optional list of context generators to run before rendering"},"renderer":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/QuestionRenderer"},{"$ref":"#/components/schemas/MockTransformConfig"}],"discriminator":{"propertyName":"config_type","mapping":{"MOCK":"#/components/schemas/MockTransformConfig","QUESTION_RENDERER":"#/components/schemas/QuestionRenderer"}}},{"type":"null"}],"title":"Renderer","description":"Optional configuration for rendering the final prompt"},"rollout_generator":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/RolloutGenerator"},{"$ref":"#/components/schemas/MockTransformConfig"}],"discriminator":{"propertyName":"config_type","mapping":{"MOCK":"#/components/schemas/MockTransformConfig","ROLLOUT_GENERATOR":"#/components/schemas/RolloutGenerator"}}},{"type":"null"}],"title":"Rollout Generator","description":"Optional configuration for generating rollouts from multiple models"},"scorer":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/RolloutScorer"},{"$ref":"#/components/schemas/MockTransformConfig"}],"discriminator":{"propertyName":"config_type","mapping":{"MOCK":"#/components/schemas/MockTransformConfig","ROLLOUT_SCORER":"#/components/schemas/RolloutScorer"}}},{"type":"null"}],"title":"Scorer","description":"Optional configuration for scoring rollouts against ground truth"}},"type":"object","title":"QuestionPipeline"},"BigQuerySeedGenerator":{"properties":{"config_type":{"type":"string","const":"BIGQUERY_SEED_GENERATOR","title":"Config Type","description":"Type of transform configuration","default":"BIGQUERY_SEED_GENERATOR"},"query":{"type":"string","title":"Query","description":"BigQuery SQL query to execute"},"seed_text_column":{"type":"string","title":"Seed Text Column","description":"Column name for Seed.seed_text","default":"text"},"date_column":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Date Column","description":"Column name for Seed.seed_creation_date"},"max_rows":{"type":"integer","title":"Max Rows","description":"Total rows to fetch across all batches","default":10000}},"type":"object","required":["query"],"title":"BigQuerySeedGenerator"},"MockTransformConfig":{"properties":{"config_type":{"type":"string","const":"MOCK","title":"Config Type","default":"MOCK"},"num_seeds":{"type":"integer","title":"Num Seeds","description":"Number of mock seeds to generate (0 = not a seed generator)","default":0},"seed_text_template":{"type":"string","title":"Seed Text Template","description":"Template for seed text","default":"Mock seed content {index}"},"delay_seconds":{"type":"number","title":"Delay Seconds","description":"Simulated processing time per item","default":0.1},"delay_variance":{"type":"number","title":"Delay Variance","description":"Random variance (+/- seconds)","default":0.05},"error_rate":{"type":"number","maximum":1,"minimum":0,"title":"Error Rate","description":"Probability of error (0.0-1.0)","default":0.01},"error_message":{"type":"string","title":"Error Message","description":"Error message when error_rate triggers","default":"Mock error"},"expansion_factor":{"type":"integer","minimum":1,"title":"Expansion Factor","description":"Outputs per input (1:N expansion)","default":1},"filter_rate":{"type":"number","maximum":1,"minimum":0,"title":"Filter Rate","description":"Probability of marking invalid","default":0},"add_question":{"type":"boolean","title":"Add Question","description":"Add a mock question to samples","default":false},"question_template":{"type":"string","title":"Question Template","description":"Template for question text","default":"Mock question {index}?"},"metadata_additions":{"additionalProperties":true,"type":"object","title":"Metadata Additions","description":"Added to sample.meta"},"estimated_input_tokens":{"type":"integer","title":"Estimated Input Tokens","description":"Estimated input tokens per call","default":0},"estimated_output_tokens":{"type":"integer","title":"Estimated Output Tokens","description":"Estimated output tokens per call","default":0},"estimated_model_name":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Estimated Model Name","description":"Model name for cost estimation"},"simulated_cost_per_call":{"type":"number","title":"Simulated Cost Per Call","description":"Simulated cost per call in dollars (records usage event when > 0)","default":0},"random_seed":{"anyOf":[{"type":"integer"},{"type":"null"}],"title":"Random Seed","description":"Seed for deterministic behavior"}},"type":"object","title":"MockTransformConfig"},"TemplateQuestionGenerator":{"properties":{"config_type":{"type":"string","const":"TEMPLATE_QUESTION_GENERATOR","title":"Config Type","description":"Type of transform configuration","default":"TEMPLATE_QUESTION_GENERATOR"},"question_template":{"type":"string","title":"Question Template","description":"Template string with {field} placeholders. Supports {seed_text}, {seed}, and any key in {meta} (e.g. {meta.label})."}},"type":"object","required":["question_template"],"title":"TemplateQuestionGenerator"},"WebSearchLabeler":{"properties":{"config_type":{"type":"string","const":"WEB_SEARCH_LABELER","title":"Config Type","description":"Type of transform configuration","default":"WEB_SEARCH_LABELER"},"confidence_threshold":{"type":"number","title":"Confidence Threshold","description":"Minimum confidence threshold for including questions","default":0.9},"answer_type":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/BinaryAnswerType"},{"$ref":"#/components/schemas/MultipleChoiceAnswerType"},{"$ref":"#/components/schemas/ContinuousAnswerType"},{"$ref":"#/components/schemas/FreeResponseAnswerType"}],"discriminator":{"propertyName":"answer_type","mapping":{"BINARY":"#/components/schemas/BinaryAnswerType","CONTINUOUS":"#/components/schemas/ContinuousAnswerType","FREE_RESPONSE":"#/components/schemas/FreeResponseAnswerType","MULTIPLE_CHOICE":"#/components/schemas/MultipleChoiceAnswerType"}}},{"type":"null"}],"title":"Answer Type","description":"The type of answer expected, used to guide the labeler"},"resolve_redirects":{"type":"boolean","title":"Resolve Redirects","description":"Resolve redirect URLs to actual destinations","default":false}},"type":"object","title":"WebSearchLabeler"},"FileSetRAGLabeler":{"properties":{"config_type":{"type":"string","const":"FILESET_RAG_LABELER","title":"Config Type","description":"Type of transform configuration","default":"FILESET_RAG_LABELER"},"file_set_id":{"type":"string","title":"File Set Id","description":"FileSet ID to query for labeling"},"metadata_filter_keys":{"items":{"type":"string"},"type":"array","title":"Metadata Filter Keys","description":"Keys from seed's file_metadata for dynamic filtering (e.g., ['ticker'])"},"metadata_filter":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Metadata Filter","description":"Static AIP-160 metadata filter (combined with dynamic via AND)"},"system_instruction":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"System Instruction","description":"Optional system instruction for the Gemini RAG query"},"confidence_threshold":{"type":"number","title":"Confidence Threshold","description":"Minimum confidence threshold for including questions","default":0.9},"answer_type":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/BinaryAnswerType"},{"$ref":"#/components/schemas/MultipleChoiceAnswerType"},{"$ref":"#/components/schemas/ContinuousAnswerType"},{"$ref":"#/components/schemas/FreeResponseAnswerType"}],"discriminator":{"propertyName":"answer_type","mapping":{"BINARY":"#/components/schemas/BinaryAnswerType","CONTINUOUS":"#/components/schemas/ContinuousAnswerType","FREE_RESPONSE":"#/components/schemas/FreeResponseAnswerType","MULTIPLE_CHOICE":"#/components/schemas/MultipleChoiceAnswerType"}}},{"type":"null"}],"title":"Answer Type","description":"The type of answer expected, used to guide the labeler"},"temporal_constraint":{"anyOf":[{"$ref":"#/components/schemas/TemporalConstraint"},{"type":"null"}],"description":"Filter documents by date relative to seed date. AFTER (>) for resolution docs, BEFORE (<=) for historical only."},"date_metadata_key":{"type":"string","title":"Date Metadata Key","description":"Gemini metadata key storing unix timestamp for temporal filtering","default":"file_date"},"model":{"type":"string","title":"Model","description":"Gemini model for RAG query","default":"gemini-3-flash-preview"}},"type":"object","required":["file_set_id"],"title":"FileSetRAGLabeler","description":"Configuration for FileSet RAG Labeler transform."},"TemporalConstraint":{"type":"string","enum":["BEFORE","AFTER"],"title":"TemporalConstraint","description":"Temporal filtering direction relative to the seed document's date.\n\nUses the `file_date` metadata key (unix timestamp int) stored on each\nGemini document by fileset_file_processor.\n\nBEFORE: file_date <= seed_timestamp  (context: no lookahead bias)\nAFTER:  file_date >  seed_timestamp  (labels: find resolutions)"},"NewsContextGenerator":{"properties":{"config_type":{"type":"string","const":"NEWS_CONTEXT_GENERATOR","title":"Config Type","description":"Type of transform configuration","default":"NEWS_CONTEXT_GENERATOR"},"num_search_queries":{"type":"integer","title":"Num Search Queries","description":"Number of search queries to generate per question","default":5},"articles_per_query":{"type":"integer","title":"Articles Per Query","description":"Number of news articles to return per search query","default":3},"num_articles":{"type":"integer","title":"Num Articles","description":"Maximum number of news articles to include in final output","default":10},"relevance_threshold":{"type":"integer","title":"Relevance Threshold","description":"Minimum relevance rating (1-6 scale) to include article","default":2},"min_articles":{"type":"integer","title":"Min Articles","description":"Minimum number of articles to ensure","default":6},"time_delta_days":{"type":"integer","title":"Time Delta Days","description":"Number of days to look back for news articles","default":30},"enable_relevance_ranking":{"type":"boolean","title":"Enable Relevance Ranking","description":"Whether to perform LLM-based relevance ranking","default":true}},"type":"object","title":"NewsContextGenerator"},"FileSetContextGenerator":{"properties":{"config_type":{"type":"string","const":"FILESET_CONTEXT_GENERATOR","title":"Config Type","description":"Type of transform configuration","default":"FILESET_CONTEXT_GENERATOR"},"file_set_id":{"type":"string","title":"File Set Id","description":"FileSet ID to query for context"},"metadata_filter_keys":{"items":{"type":"string"},"type":"array","title":"Metadata Filter Keys","description":"Keys from seed's file_metadata for dynamic filtering (e.g., ['ticker'])"},"metadata_filter":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Metadata Filter","description":"Static AIP-160 metadata filter (combined with dynamic via AND)"},"system_instruction":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"System Instruction","description":"Optional system instruction for the Gemini model"},"query_template":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Query Template","description":"Template with {question} placeholder; default: raw question text"},"temporal_constraint":{"anyOf":[{"$ref":"#/components/schemas/TemporalConstraint"},{"type":"null"}],"description":"Filter documents by date relative to seed date. BEFORE (<=) for context without lookahead, AFTER (>) for future docs."},"date_metadata_key":{"type":"string","title":"Date Metadata Key","description":"Gemini metadata key storing unix timestamp for temporal filtering","default":"file_date"},"model":{"type":"string","title":"Model","description":"Gemini model to use for file search","default":"gemini-3-flash-preview"}},"type":"object","required":["file_set_id"],"title":"FileSetContextGenerator","description":"Configuration for FileSet Context Generator transform."},"QuestionRenderer":{"properties":{"config_type":{"type":"string","const":"QUESTION_RENDERER","title":"Config Type","description":"Type of transform configuration","default":"QUESTION_RENDERER"},"template":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Template","description":"Custom template for rendering the prompt. If not provided, dynamically builds based on available content. Supports placeholders like {question_text}, {context}, {answer_instructions}."},"answer_type":{"anyOf":[{"oneOf":[{"$ref":"#/components/schemas/BinaryAnswerType"},{"$ref":"#/components/schemas/MultipleChoiceAnswerType"},{"$ref":"#/components/schemas/ContinuousAnswerType"},{"$ref":"#/components/schemas/FreeResponseAnswerType"}],"discriminator":{"propertyName":"answer_type","mapping":{"BINARY":"#/components/schemas/BinaryAnswerType","CONTINUOUS":"#/components/schemas/ContinuousAnswerType","FREE_RESPONSE":"#/components/schemas/FreeResponseAnswerType","MULTIPLE_CHOICE":"#/components/schemas/MultipleChoiceAnswerType"}}},{"type":"null"}],"title":"Answer Type","description":"The type of answer expected, used to render answer instructions"}},"type":"object","title":"QuestionRenderer"},"RolloutGenerator":{"properties":{"config_type":{"type":"string","const":"ROLLOUT_GENERATOR","title":"Config Type","default":"ROLLOUT_GENERATOR"},"models":{"items":{"$ref":"#/components/schemas/ModelConfig"},"type":"array","title":"Models","description":"Model names or ModelConfig objects"},"prompt_template":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Prompt Template","description":"Prompt template with {column} placeholders. If None, uses sample.prompt"},"input_columns":{"items":{"type":"string"},"type":"array","title":"Input Columns","description":"Columns to substitute into template (from meta)"},"output_schema":{"anyOf":[{},{"type":"null"}],"title":"Output Schema","description":"Pydantic model for structured output"}},"type":"object","required":["models"],"title":"RolloutGenerator"},"ModelConfig":{"properties":{"model_name":{"type":"string","title":"Model Name","default":"meta-llama/llama-3.3-70b-instruct"},"model_source":{"$ref":"#/components/schemas/ModelSourceType","default":"OPEN_ROUTER"},"temperature":{"type":"number","title":"Temperature","default":1},"max_tokens":{"anyOf":[{"type":"integer"},{"type":"null"}],"title":"Max Tokens"},"ip_address":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Ip Address"},"lora_base_model_name":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Lora Base Model Name"},"lora_repo_path":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Lora Repo Path"},"lora_checkpoint_path":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Lora Checkpoint Path"},"runpod_endpoint_id":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Runpod Endpoint Id"},"is_lightningrod_model":{"anyOf":[{"type":"boolean"},{"type":"null"}],"title":"Is Lightningrod Model"},"openrouter_provider":{"anyOf":[{"items":{"type":"string"},"type":"array"},{"type":"null"}],"title":"Openrouter Provider"},"reasoning_effort":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Reasoning Effort"},"is_reasoning_model":{"anyOf":[{"type":"boolean"},{"type":"null"}],"title":"Is Reasoning Model"},"disable_reasoning":{"type":"boolean","title":"Disable Reasoning","default":false},"use_pipeline_key":{"type":"boolean","title":"Use Pipeline Key","default":false}},"type":"object","title":"ModelConfig"},"ModelSourceType":{"type":"string","enum":["OPENAI","VLLM","DIRECT_COMPLETIONS","OPEN_ROUTER","HUGGINGFACE_ENDPOINTS","DEEPSEEK","DEEPINFRA","RUNPOD","SGLANG","TINKER","MODAL_LORA_VLLM"],"title":"ModelSourceType"},"RolloutScorer":{"properties":{"config_type":{"type":"string","const":"ROLLOUT_SCORER","title":"Config Type","default":"ROLLOUT_SCORER"},"answer_type":{"oneOf":[{"$ref":"#/components/schemas/BinaryAnswerType"},{"$ref":"#/components/schemas/MultipleChoiceAnswerType"},{"$ref":"#/components/schemas/ContinuousAnswerType"},{"$ref":"#/components/schemas/FreeResponseAnswerType"}],"title":"Answer Type","description":"the answer type of the question","discriminator":{"propertyName":"answer_type","mapping":{"BINARY":"#/components/schemas/BinaryAnswerType","CONTINUOUS":"#/components/schemas/ContinuousAnswerType","FREE_RESPONSE":"#/components/schemas/FreeResponseAnswerType","MULTIPLE_CHOICE":"#/components/schemas/MultipleChoiceAnswerType"}}},"multiple_choice_options":{"anyOf":[{"additionalProperties":{"type":"string"},"type":"object"},{"type":"null"}],"title":"Multiple Choice Options"},"is_mutually_exclusive":{"type":"boolean","title":"Is Mutually Exclusive","default":true}},"type":"object","required":["answer_type"],"title":"RolloutScorer"},"EstimateCostResponse":{"properties":{"total_cost_dollars":{"type":"number","title":"Total Cost Dollars"},"llm_cost_dollars":{"type":"number","title":"Llm Cost Dollars"},"web_search_cost_dollars":{"type":"number","title":"Web Search Cost Dollars"},"url_download_cost_dollars":{"type":"number","title":"Url Download Cost Dollars"},"gemini_grounding_cost_dollars":{"type":"number","title":"Gemini Grounding Cost Dollars","default":0},"bigquery_cost_dollars":{"type":"number","title":"Bigquery Cost Dollars","default":0},"usage":{"$ref":"#/components/schemas/UsageSummary"},"steps":{"items":{"$ref":"#/components/schemas/StepCostBreakdown"},"type":"array","title":"Steps"}},"type":"object","required":["total_cost_dollars","llm_cost_dollars","web_search_cost_dollars","url_download_cost_dollars","usage","steps"],"title":"EstimateCostResponse"},"UsageSummary":{"properties":{"events":{"additionalProperties":{"$ref":"#/components/schemas/EventUsageSummary"},"type":"object","title":"Events","default":{}},"llm_by_model":{"additionalProperties":{"$ref":"#/components/schemas/LLMModelUsageSummary"},"type":"object","title":"Llm By Model","default":{}},"total_cost":{"type":"number","title":"Total Cost","default":0}},"type":"object","title":"UsageSummary","description":"Flexible usage statistics by event type and LLM model."},"EventUsageSummary":{"properties":{"count":{"type":"integer","title":"Count","default":0},"amount":{"type":"number","title":"Amount","default":0},"cost":{"type":"number","title":"Cost","default":0},"cached_count":{"type":"integer","title":"Cached Count","default":0}},"type":"object","title":"EventUsageSummary","description":"Usage for a single event type."},"LLMModelUsageSummary":{"properties":{"count":{"type":"integer","title":"Count","default":0},"input_tokens":{"type":"integer","title":"Input Tokens","default":0},"output_tokens":{"type":"integer","title":"Output Tokens","default":0},"cost":{"type":"number","title":"Cost","default":0},"cached_count":{"type":"integer","title":"Cached Count","default":0}},"type":"object","title":"LLMModelUsageSummary","description":"Usage for a single LLM model."},"StepCostBreakdown":{"properties":{"step_name":{"type":"string","title":"Step Name"},"step_type":{"$ref":"#/components/schemas/TransformType"},"total_cost_dollars":{"type":"number","title":"Total Cost Dollars"},"usage":{"$ref":"#/components/schemas/UsageSummary"},"output_count":{"type":"integer","title":"Output Count"},"cost_per_output":{"type":"number","title":"Cost Per Output"}},"type":"object","required":["step_name","step_type","total_cost_dollars","usage","output_count","cost_per_output"],"title":"StepCostBreakdown"},"TransformType":{"type":"string","enum":["NEWS_SEED_GENERATOR","GDELT_SEED_GENERATOR","QUESTION_GENERATOR","QUESTION_AND_LABEL_GENERATOR","QUESTION_FILTER","FORWARD_LOOKING_QUESTION_GENERATOR","QUESTION_QUALITY_SCORER","SCORE_FILTER","FILTER_CRITERIA","WEB_SEARCH_LABELER","NEWS_CONTEXT_GENERATOR","PIPELINE","QUESTION_PIPELINE","QUESTION_RENDERER","ROLLOUT_GENERATOR","BIGQUERY_SEED_GENERATOR","FILESET_SEED_GENERATOR","FILESET_QUERY_SEED_GENERATOR","TEMPLATE_QUESTION_GENERATOR","ROLLOUT_SCORER","FILESET_CONTEXT_GENERATOR","FILESET_RAG_LABELER","MOCK"],"title":"TransformType"},"HTTPValidationError":{"properties":{"detail":{"items":{"$ref":"#/components/schemas/ValidationError"},"type":"array","title":"Detail"}},"type":"object","title":"HTTPValidationError"},"ValidationError":{"properties":{"loc":{"items":{"anyOf":[{"type":"string"},{"type":"integer"}]},"type":"array","title":"Location"},"msg":{"type":"string","title":"Message"},"type":{"type":"string","title":"Error Type"},"input":{"title":"Input"},"ctx":{"type":"object","title":"Context"}},"type":"object","required":["loc","msg","type"],"title":"ValidationError"}}},"paths":{"/transform-jobs/cost-estimation":{"post":{"tags":["Transform Jobs"],"summary":"Cost Estimation","description":"Estimate maximum cost for a transform job before execution","operationId":"cost_estimation_transform_jobs_cost_estimation_post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/EstimateCostRequest"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/EstimateCostResponse"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}}}}
```

## Get Transform Job

> Get a transform job by ID

```json
{"openapi":"3.1.0","info":{"title":"LightningRod API","version":"1.0.0"},"servers":[{"url":"/api/public/v1"}],"security":[{"HTTPBearer":[]}],"components":{"securitySchemes":{"HTTPBearer":{"type":"http","scheme":"bearer"}},"schemas":{"TransformJob":{"properties":{"id":{"type":"string","title":"Id"},"organization_id":{"type":"string","title":"Organization Id"},"configuration_id":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Configuration Id"},"name":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Name"},"status":{"$ref":"#/components/schemas/TransformJobStatus"},"error_message":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Error Message"},"warning_message":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Warning Message"},"modal_function_call_id":{"type":"string","title":"Modal Function Call Id"},"modal_app_id":{"type":"string","title":"Modal App Id"},"transform_config":{"type":"string","title":"Transform Config"},"input_dataset_id":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Input Dataset Id"},"output_dataset_id":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Output Dataset Id"},"usage":{"anyOf":[{"$ref":"#/components/schemas/JobUsage"},{"type":"null"}]},"estimated_cost_dollars":{"anyOf":[{"type":"number"},{"type":"null"}],"title":"Estimated Cost Dollars"},"created_at":{"type":"string","format":"date-time","title":"Created At"},"updated_at":{"type":"string","format":"date-time","title":"Updated At"}},"type":"object","required":["id","organization_id","status","modal_function_call_id","modal_app_id","transform_config","input_dataset_id","output_dataset_id","created_at","updated_at"],"title":"TransformJob"},"TransformJobStatus":{"type":"string","enum":["RUNNING","COMPLETED","FAILED","CANCELLED"],"title":"TransformJobStatus"},"JobUsage":{"properties":{"total":{"anyOf":[{"$ref":"#/components/schemas/UsageSummary"},{"type":"null"}]},"by_step":{"anyOf":[{"additionalProperties":{"$ref":"#/components/schemas/UsageSummary"},"type":"object"},{"type":"null"}],"title":"By Step"},"max_cost_dollars":{"anyOf":[{"type":"number"},{"type":"null"}],"title":"Max Cost Dollars"},"current_cost_dollars":{"anyOf":[{"type":"number"},{"type":"null"}],"title":"Current Cost Dollars"},"estimated_cost_dollars":{"anyOf":[{"type":"number"},{"type":"null"}],"title":"Estimated Cost Dollars"},"pipeline_summary":{"anyOf":[{"items":{"$ref":"#/components/schemas/PipelineStepSummary"},"type":"array"},{"type":"null"}],"title":"Pipeline Summary"}},"type":"object","title":"JobUsage","description":"Aggregated usage statistics for a transform job with step breakdown."},"UsageSummary":{"properties":{"events":{"additionalProperties":{"$ref":"#/components/schemas/EventUsageSummary"},"type":"object","title":"Events","default":{}},"llm_by_model":{"additionalProperties":{"$ref":"#/components/schemas/LLMModelUsageSummary"},"type":"object","title":"Llm By Model","default":{}},"total_cost":{"type":"number","title":"Total Cost","default":0}},"type":"object","title":"UsageSummary","description":"Flexible usage statistics by event type and LLM model."},"EventUsageSummary":{"properties":{"count":{"type":"integer","title":"Count","default":0},"amount":{"type":"number","title":"Amount","default":0},"cost":{"type":"number","title":"Cost","default":0},"cached_count":{"type":"integer","title":"Cached Count","default":0}},"type":"object","title":"EventUsageSummary","description":"Usage for a single event type."},"LLMModelUsageSummary":{"properties":{"count":{"type":"integer","title":"Count","default":0},"input_tokens":{"type":"integer","title":"Input Tokens","default":0},"output_tokens":{"type":"integer","title":"Output Tokens","default":0},"cost":{"type":"number","title":"Cost","default":0},"cached_count":{"type":"integer","title":"Cached Count","default":0}},"type":"object","title":"LLMModelUsageSummary","description":"Usage for a single LLM model."},"PipelineStepSummary":{"properties":{"step_index":{"type":"integer","title":"Step Index"},"transform_name":{"type":"string","title":"Transform Name"},"input_count":{"type":"integer","title":"Input Count","default":0},"output_count":{"type":"integer","title":"Output Count","default":0},"rejected_count":{"type":"integer","title":"Rejected Count","default":0},"error_count":{"type":"integer","title":"Error Count","default":0},"rejection_reasons":{"additionalProperties":{"type":"integer"},"type":"object","title":"Rejection Reasons","default":{}}},"type":"object","required":["step_index","transform_name"],"title":"PipelineStepSummary","description":"Summary of a single pipeline step including rejection reasons."},"HTTPValidationError":{"properties":{"detail":{"items":{"$ref":"#/components/schemas/ValidationError"},"type":"array","title":"Detail"}},"type":"object","title":"HTTPValidationError"},"ValidationError":{"properties":{"loc":{"items":{"anyOf":[{"type":"string"},{"type":"integer"}]},"type":"array","title":"Location"},"msg":{"type":"string","title":"Message"},"type":{"type":"string","title":"Error Type"},"input":{"title":"Input"},"ctx":{"type":"object","title":"Context"}},"type":"object","required":["loc","msg","type"],"title":"ValidationError"}}},"paths":{"/transform-jobs/{job_id}":{"get":{"tags":["Transform Jobs"],"summary":"Get Transform Job","description":"Get a transform job by ID","operationId":"get_transform_job_transform_jobs__job_id__get","parameters":[{"name":"job_id","in":"path","required":true,"schema":{"type":"string","title":"Job Id"}}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TransformJob"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}}}}
```

## Cancel Transform Job

> Cancel a running transform job

```json
{"openapi":"3.1.0","info":{"title":"LightningRod API","version":"1.0.0"},"servers":[{"url":"/api/public/v1"}],"security":[{"HTTPBearer":[]}],"components":{"securitySchemes":{"HTTPBearer":{"type":"http","scheme":"bearer"}},"schemas":{"HTTPValidationError":{"properties":{"detail":{"items":{"$ref":"#/components/schemas/ValidationError"},"type":"array","title":"Detail"}},"type":"object","title":"HTTPValidationError"},"ValidationError":{"properties":{"loc":{"items":{"anyOf":[{"type":"string"},{"type":"integer"}]},"type":"array","title":"Location"},"msg":{"type":"string","title":"Message"},"type":{"type":"string","title":"Error Type"},"input":{"title":"Input"},"ctx":{"type":"object","title":"Context"}},"type":"object","required":["loc","msg","type"],"title":"ValidationError"}}},"paths":{"/transform-jobs/{job_id}":{"delete":{"tags":["Transform Jobs"],"summary":"Cancel Transform Job","description":"Cancel a running transform job","operationId":"cancel_transform_job_transform_jobs__job_id__delete","parameters":[{"name":"job_id","in":"path","required":true,"schema":{"type":"string","title":"Job Id"}}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"400":{"description":"Cannot cancel job with current status"},"404":{"description":"Transform job not found"},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}}}}
```

## Get Transform Job Metrics

> Get pipeline metrics for a transform job

```json
{"openapi":"3.1.0","info":{"title":"LightningRod API","version":"1.0.0"},"servers":[{"url":"/api/public/v1"}],"security":[{"HTTPBearer":[]}],"components":{"securitySchemes":{"HTTPBearer":{"type":"http","scheme":"bearer"}},"schemas":{"PipelineMetricsResponse":{"properties":{"total_input_rows":{"type":"integer","title":"Total Input Rows"},"total_output_rows":{"type":"integer","title":"Total Output Rows"},"total_duration_seconds":{"type":"number","title":"Total Duration Seconds"},"steps":{"items":{"$ref":"#/components/schemas/TransformStepMetricsResponse"},"type":"array","title":"Steps"}},"type":"object","required":["total_input_rows","total_output_rows","total_duration_seconds","steps"],"title":"PipelineMetricsResponse"},"TransformStepMetricsResponse":{"properties":{"step_index":{"type":"integer","title":"Step Index"},"transform_name":{"type":"string","title":"Transform Name"},"input_rows":{"type":"integer","title":"Input Rows"},"output_rows":{"type":"integer","title":"Output Rows"},"rejected_count":{"type":"integer","title":"Rejected Count"},"error_count":{"type":"integer","title":"Error Count"},"duration_seconds":{"type":"number","title":"Duration Seconds"},"progress":{"type":"number","title":"Progress"},"summary":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Summary"}},"type":"object","required":["step_index","transform_name","input_rows","output_rows","rejected_count","error_count","duration_seconds","progress","summary"],"title":"TransformStepMetricsResponse"},"HTTPValidationError":{"properties":{"detail":{"items":{"$ref":"#/components/schemas/ValidationError"},"type":"array","title":"Detail"}},"type":"object","title":"HTTPValidationError"},"ValidationError":{"properties":{"loc":{"items":{"anyOf":[{"type":"string"},{"type":"integer"}]},"type":"array","title":"Location"},"msg":{"type":"string","title":"Message"},"type":{"type":"string","title":"Error Type"},"input":{"title":"Input"},"ctx":{"type":"object","title":"Context"}},"type":"object","required":["loc","msg","type"],"title":"ValidationError"}}},"paths":{"/transform-jobs/{job_id}/metrics":{"get":{"tags":["Transform Jobs"],"summary":"Get Transform Job Metrics","description":"Get pipeline metrics for a transform job","operationId":"get_transform_job_metrics_transform_jobs__job_id__metrics_get","parameters":[{"name":"job_id","in":"path","required":true,"schema":{"type":"string","title":"Job Id"}}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/PipelineMetricsResponse"}}}},"404":{"description":"Pipeline metrics not found"},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}}}}
```


---

# Agent Instructions: Querying This Documentation

If you need additional information that is not directly available in this page, you can query the documentation dynamically by asking a question.

Perform an HTTP GET request on the current page URL with the `ask` query parameter:

```
GET https://docs.lightningrod.ai/rest-api/transform-jobs.md?ask=<question>
```

The question should be specific, self-contained, and written in natural language.
The response will contain a direct answer to the question and relevant excerpts and sources from the documentation.

Use this mechanism when the answer is not explicitly present in the current page, you need clarification or additional context, or you want to retrieve related documentation sections.