schemas
Every request and response body the endpoints above name, as JSON Schema generated from the Rust types they deserialize into.
ApiError
The error body every failing request comes back with.
A Rust type rather than a hand-written schema because it has to stay in step with what AppError actually serializes — an_error_body_matches_the_documented_shape in tests/api.rs is what says so.
schema
json
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "The error body every failing request comes back with.\n\nA Rust type rather than a hand-written schema because it has to stay in step\nwith what `AppError` actually serializes — `an_error_body_matches_the_documented_shape`\nin `tests/api.rs` is what says so.",
"properties": {
"error": {
"description": "The error, as one line. The line includes the cause, in the form\n`context: cause`.",
"type": "string"
}
},
"required": [
"error"
],
"title": "ApiError",
"type": "object"
}AuthDto
The caller, and whether the server checks credentials.
schema
json
{
"$defs": {
"Role": {
"description": "The operations that an account can do. A role applies to all pipelines and\nall connections.",
"oneOf": [
{
"const": "admin",
"description": "All operations. For example, create and delete pipelines and\nconnections, save and revert the config file, and change the layout.",
"type": "string"
},
{
"const": "read",
"description": "Read all data and change nothing. This is the default.",
"type": "string"
}
]
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "The caller, and whether the server checks credentials.",
"properties": {
"authentication_required": {
"description": "Whether the server checks credentials. It is `false` on a server\nstarted without `--server-config`, or with `auth: {type: none}`.",
"type": "boolean"
},
"role": {
"anyOf": [
{
"$ref": "#/$defs/Role"
},
{
"type": "null"
}
],
"description": "The role of the caller. `null` means signed out. This is different\nfrom `read`: a `read` user can see the graph, and a signed-out caller\ncannot."
},
"username": {
"description": "The signed-in user, or `null` for a caller with no credentials.",
"type": [
"string",
"null"
]
}
},
"required": [
"authentication_required"
],
"title": "AuthDto",
"type": "object"
}BucketContents
The contents of a bucket.
schema
json
{
"$defs": {
"BucketEntry": {
"description": "One key's contents as the API reports them.",
"properties": {
"key": {
"type": "string"
},
"updated_at": {
"type": "string"
},
"values": {
"additionalProperties": true,
"type": "object"
}
},
"required": [
"key",
"values",
"updated_at"
],
"type": "object"
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "The contents of a bucket.",
"properties": {
"entries": {
"items": {
"$ref": "#/$defs/BucketEntry"
},
"type": "array"
},
"keys": {
"format": "uint",
"minimum": 0,
"type": "integer"
},
"name": {
"type": "string"
},
"truncated": {
"description": "True when `entries` has fewer entries than `keys`, because the response\nhas a limit.",
"type": "boolean"
}
},
"required": [
"name",
"keys",
"entries",
"truncated"
],
"title": "BucketContents",
"type": "object"
}BucketSummary
schema
json
{
"$defs": {
"BucketSummary": {
"description": "One bucket as the API reports it.",
"properties": {
"idle_timeout_secs": {
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"keys": {
"description": "The number of keys in the bucket now. The number includes expired keys.\nkayak removes them on the next write.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"max_keys": {
"format": "uint",
"minimum": 0,
"type": "integer"
},
"name": {
"type": "string"
}
},
"required": [
"name",
"keys",
"max_keys"
],
"type": "object"
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema",
"items": {
"$ref": "#/$defs/BucketSummary"
},
"title": "Array_of_BucketSummary",
"type": "array"
}ComponentDoc
One component: everything /docs shows about it.
schema
json
{
"$defs": {
"Family": {
"description": "Which plugin point a component plugs into. Also the grouping the docs\nsidebar uses, which is why it's ordered the way a pipeline reads.\n\nA connection isn't a stage of a pipeline, but it is configured the same way\n— a tagged struct with doc-commented fields — so it documents itself and\ngenerates its form through exactly this machinery.",
"enum": [
"input",
"transform",
"output",
"connection"
],
"type": "string"
},
"FieldDoc": {
"description": "One configurable field of a component.",
"properties": {
"description": {
"description": "The field's doc comment, if it has one.",
"type": [
"string",
"null"
]
},
"field_type": {
"$ref": "#/$defs/FieldType",
"description": "The same thing in a form a UI can dispatch on."
},
"name": {
"description": "The wire name — what actually goes in the JSON.",
"type": "string"
},
"required": {
"description": "Required fields have to appear in the JSON; optional ones may be omitted.",
"type": "boolean"
},
"type_name": {
"description": "A rendered type, e.g. `string`, `integer`, or `sum | avg | min | max`\nfor a field that only accepts certain values.",
"type": "string"
}
},
"required": [
"name",
"type_name",
"field_type",
"required"
],
"type": "object"
},
"FieldType": {
"description": "What a field actually accepts, as something a form can be built from.\n\n[`FieldDoc::type_name`] is the same information rendered for a human to\nread; this is the machine-readable half, and it is what the \"add pipeline\"\nmodal picks a widget and a validation rule from. Keeping it here rather\nthan in the frontend is the same bargain as the rest of this module: the\nconfig schema stays the single source of truth, so a new field gets a\nworking form control without anyone editing the UI.",
"oneOf": [
{
"enum": [
"text",
"integer",
"number",
"boolean"
],
"type": "string"
},
{
"additionalProperties": false,
"description": "A closed set of accepted values — rendered as a dropdown, not a box.",
"properties": {
"enum": {
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"enum"
],
"type": "object"
},
{
"const": "pipeline_id",
"description": "The id of another pipeline. A string on the wire, but the set of valid\nanswers is the graph the server is currently running, which only the UI\nknows — so it renders as a dropdown of the pipelines that exist rather\nthan as a box to retype an id into.",
"type": "string"
},
{
"additionalProperties": false,
"description": "The name of a configured connection, of the kind carried here (`kafka`,\n`nats`, ...). Like [`FieldType::PipelineId`] it is a string on the wire\nwhose valid answers are server state, so it renders as a dropdown — of\nthe connections of *that kind*, since a nats connection is no use to a\nkafka input.",
"properties": {
"connection": {
"type": "string"
}
},
"required": [
"connection"
],
"type": "object"
},
{
"const": "message_field",
"description": "The name of a field *in the messages* — a column's `field`, a filter's\ncomparison, an aggregation's source. A string on the wire like\n[`FieldType::PipelineId`], and a separate type for the same reason:\nwhat the control should be is not derivable from the wire type.\n\nUnlike the two dropdowns above this one is a **box with suggestions**,\nnever a closed list. The suggestions come from a sample of real\nmessages ([`crate::schema::infer`]), and a sample is a handful of\nmessages rather than a schema — a stream that carries `error_code` only\nwhen something breaks would otherwise have no way to say so.",
"type": "string"
},
{
"additionalProperties": false,
"description": "A value that is one of several shapes, tagged by one of its own\nproperties — an input's `buffer`, which is `{\"type\": \"static\", \"size\":\n10}` or `{\"type\": \"tumbling\", \"window_seconds\": 30}`.\n\nThis is the field-level twin of [`ComponentDoc::variants`], and it is\nwhat makes a form conditional: which fields a value has depends on which\nvariant was picked, so the tag is chosen first and the rest of the form\nfollows from it.",
"properties": {
"union": {
"$ref": "#/$defs/UnionDoc"
}
},
"required": [
"union"
],
"type": "object"
},
{
"additionalProperties": false,
"description": "A value that is an object with a fixed set of fields of its own — a file\noutput's `rotate`. Not a choice, just a level of nesting, so the form\nrenders its fields inline.",
"properties": {
"object": {
"items": {
"$ref": "#/$defs/FieldDoc"
},
"type": "array"
}
},
"required": [
"object"
],
"type": "object"
},
{
"additionalProperties": false,
"description": "A value that is a list of some other type — a reducer's `aggregations`.\nUnlike every other field type this one has no fixed number of boxes, so\nthe form renders rows that can be added and taken away, each of them\nwhatever the element type asks for.\n\nThe element is carried as a whole [`FieldDoc`] because that is what a\ncontrol is chosen from, and its [`FieldDoc::name`] is empty: a list\nelement has no name of its own, it has a position, which only the form\nrendering it knows. It is always required — a row that is there is a\nvalue that will be sent.",
"properties": {
"list": {
"$ref": "#/$defs/FieldDoc"
}
},
"required": [
"list"
],
"type": "object"
},
{
"additionalProperties": false,
"description": "Source code in some language, carried as a string — a `script`\ntransform's `code`. The language is the payload (`rhai`), because a form\ncannot syntax-highlight what it does not know the name of.\n\nA string on the wire like [`FieldType::Text`], and a separate type for\nthe reason [`FieldType::PipelineId`] is: what the *control* should be is\nnot derivable from the wire type. A one-line box is the wrong shape for\ntwenty lines of code, and tab is a character rather than a way out of\nthe field.",
"properties": {
"script": {
"type": "string"
}
},
"required": [
"script"
],
"type": "object"
},
{
"const": "json",
"description": "Something with a shape of its own that is none of the above — a union\ntagged in a spelling this module doesn't read, say. There is no general\nwidget for those, so the form takes them as literal JSON.",
"type": "string"
}
]
},
"MetaFieldDoc": {
"description": "One metadata field that an input attaches, and its contents.",
"properties": {
"description": {
"type": "string"
},
"name": {
"description": "The name of the field in the metadata object.",
"type": "string"
}
},
"required": [
"name",
"description"
],
"type": "object"
},
"UnionDoc": {
"description": "A tagged union as a form can render it: pick the tag, then fill in whatever\nthat variant asks for.",
"properties": {
"tag": {
"description": "The property that says which variant this is — `type`, for every union\nin the config today. Carried rather than assumed, because it is what\ngoes on the wire beside the variant's own fields.",
"type": "string"
},
"variants": {
"description": "The variants, named by their tag *value* (`static`, `tumbling`) rather\nthan by a Rust variant name. The tag itself is not among any variant's\nfields — it is the choice, not a thing to fill in.",
"items": {
"$ref": "#/$defs/VariantDoc"
},
"type": "array"
}
},
"required": [
"tag",
"variants"
],
"type": "object"
},
"VariantDoc": {
"description": "One variant's fields: a [`UnionDoc`]'s, or those of a component config that\nis a tagged enum rather than a flat struct. No component is spelled the\nsecond way today — `filter` was, until it took a list of conditions — but\nthe reflection reads the shape, and the tests pin it against a stand-in.",
"properties": {
"fields": {
"items": {
"$ref": "#/$defs/FieldDoc"
},
"type": "array"
},
"name": {
"type": "string"
}
},
"required": [
"name",
"fields"
],
"type": "object"
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "One component: everything `/docs` shows about it.",
"properties": {
"description": {
"description": "The config struct's doc comment, if it has one.",
"type": [
"string",
"null"
]
},
"family": {
"$ref": "#/$defs/Family"
},
"fields": {
"items": {
"$ref": "#/$defs/FieldDoc"
},
"type": "array"
},
"kind": {
"description": "The `type` tag that selects this component in a config file.",
"type": "string"
},
"metadata": {
"description": "What this input attaches to a message when its `envelope` is set —\nempty for every family but [`Family::Input`], and declared in\n[`crate::metadata`] rather than reflected, since a schema cannot know\nwhat a nats subscription knows.",
"items": {
"$ref": "#/$defs/MetaFieldDoc"
},
"type": "array"
},
"variants": {
"description": "Empty for all but the enum-shaped components; when it isn't, the fields\nlive on the variants instead.",
"items": {
"$ref": "#/$defs/VariantDoc"
},
"type": "array"
}
},
"required": [
"kind",
"family",
"fields",
"variants"
],
"title": "ComponentDoc",
"type": "object"
}Config
One pipeline. kayak merges all inputs into one stream. The stream goes through the transforms in sequence. Each batch that results goes to each output.
schema
json
{
"$defs": {
"AckMode": {
"description": "When an input acknowledges a message to its broker.",
"oneOf": [
{
"const": "on_receipt",
"description": "Acknowledge the message when it arrives, before the transforms and the\noutputs. The default. A crash before the output writes the message can\nlose it.",
"type": "string"
},
{
"const": "on_delivery",
"description": "Acknowledge the message when it leaves this pipeline. Each output of\nthis pipeline must return, with or without success. Each downstream\npipeline must accept the message into its queue. A failed output does\nnot stop the acknowledgement. kayak does not wait for the outputs of the\ndownstream pipelines.",
"type": "string"
}
]
},
"Aggregation": {
"description": "One value to calculate for a group, and the field name for the result.",
"properties": {
"as": {
"description": "The field that contains the result in the sent message. Each\naggregation must have a different `as`. It must not be the same as a\n`group_by` field.",
"type": "string"
},
"field": {
"description": "The field to aggregate. Required for all functions except `count`.\nWithout a `field`, `count` counts the messages.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"function": {
"$ref": "#/$defs/ReduceFnKind",
"description": "How to combine the values."
}
},
"required": [
"function",
"as"
],
"title": "aggregation",
"type": "object"
},
"ArithmeticOperator": {
"description": "What an [`Mapping::Arithmetic`] does with its two operands.",
"oneOf": [
{
"const": "add",
"description": "Left + right.",
"type": "string"
},
{
"const": "subtract",
"description": "Left − right.",
"type": "string"
},
{
"const": "multiply",
"description": "Left × right.",
"type": "string"
},
{
"const": "divide",
"description": "Left ÷ right. A literal zero on the right is an error when the pipeline\nbuilds. For a field that holds zero, `on_zero` applies.",
"type": "string"
},
{
"const": "min",
"description": "The smaller of left and right. With a literal on one side, this is an\nupper limit.",
"type": "string"
},
{
"const": "max",
"description": "The larger of left and right. With a literal on one side, this is a\nlower limit.",
"type": "string"
}
]
},
"Band": {
"description": "The power in one frequency band, as a feature.",
"properties": {
"as": {
"description": "The field to write the power of the band to.",
"type": "string"
},
"high_hz": {
"description": "The upper limit of the band, in hertz. The band does not include this\nvalue. It must be more than `low_hz`.",
"format": "double",
"type": "number"
},
"low_hz": {
"description": "The lower limit of the band, in hertz. The band includes this value.",
"format": "double",
"type": "number"
}
},
"required": [
"low_hz",
"high_hz",
"as"
],
"title": "band",
"type": "object"
},
"BufferConfig": {
"description": "How an input collects its messages into batches before the transforms.\n\nThe three types use two limits: a count, a time, or both. With both, the\nfirst limit that is reached closes the batch. A buffer never sends an empty\nbatch. The time starts when the first message of a batch arrives, so a\nquiet input sends nothing.\n\n`size` is a minimum, not a maximum. The buffer does not divide a batch\nthat arrives. Thus an input with `max_batch` can give a larger batch.",
"oneOf": [
{
"description": "Wait for a number of messages. There is no time limit.",
"properties": {
"size": {
"description": "The number of messages in a batch.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"type": {
"const": "static",
"type": "string"
}
},
"required": [
"type",
"size"
],
"type": "object"
},
{
"description": "Wait for a time. The batch contains at least one message. The time\nstarts when the first message arrives.",
"properties": {
"type": {
"const": "tumbling",
"type": "string"
},
"window_seconds": {
"description": "The time to collect messages, in s, from the first message.",
"format": "uint",
"minimum": 0,
"type": "integer"
}
},
"required": [
"type",
"window_seconds"
],
"type": "object"
},
{
"description": "Use both limits. The first limit that is reached closes the batch. Use\nthis type when the rate of the input changes. `size` sets the largest\nbatch when the input is busy. `window_seconds` sets the longest wait\nwhen the input is quiet.",
"properties": {
"size": {
"description": "The number of messages that closes the batch immediately.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"type": {
"const": "batch",
"type": "string"
},
"window_seconds": {
"description": "The maximum time to wait, in s, from the first message in the batch.",
"format": "uint",
"minimum": 0,
"type": "integer"
}
},
"required": [
"type",
"size",
"window_seconds"
],
"type": "object"
}
]
},
"BufferGateConfig": {
"description": "A condition on a state bucket that releases a `buffer` transform.\n\nThe buffer tests the conditions against the bucket entry as an object. The\nnames that `remember` wrote are its fields. `field` is a dotted path, as in\nall other transforms. All conditions must be true.\n\nThe gate applies to the full buffer. It does not test each kept message.\nWhen the gate opens, the buffer sends all kept messages.",
"properties": {
"bucket": {
"description": "The bucket to watch. The default is the bucket in the `state` of this\npipeline. A pipeline with no `state` must set it.",
"type": [
"string",
"null"
]
},
"conditions": {
"description": "The conditions that must all be true to release the buffer. At least\none condition is required.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"key": {
"description": "The key in the bucket to read. This is a literal key, not a field path.\nIf you do not set it, the gate reads the value for the full bucket.\n`remember` writes that value when the `state` of its pipeline has no\n`key`.",
"type": [
"string",
"null"
]
}
},
"required": [
"conditions"
],
"title": "buffer gate",
"type": "object"
},
"BufferTransformConfig": {
"description": "Keeps messages and sends them on when a trigger fires.\n\nThere are three triggers: a message count, a time, and a condition on a\nstate bucket. You can use them together. The first trigger that fires\nreleases the messages. A buffer with no trigger fails to build.\n\n`size` sends batches of exactly that number of messages. `seconds` and\n`until` send all messages that the buffer keeps, as one batch.\n\nThe `buffer` setting on an input is a different thing. It makes batches\nbefore the transforms. The `buffer` transform makes batches at its\nposition in the chain, for example after a `filter` or a `recall`.",
"properties": {
"max_messages": {
"description": "The maximum number of messages to keep. At this number, the buffer sends\nall kept messages and writes one warning to the log. Required if `size`\nis not set. Without a limit, a condition that is never true makes the\nbuffer use more and more memory.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"seconds": {
"description": "Send all kept messages this number of seconds after the first kept\nmessage. The time starts when the buffer keeps a message. An empty\nbuffer sends nothing.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"size": {
"description": "Send batches of exactly this number of messages, when each batch is\nfull.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"until": {
"anyOf": [
{
"$ref": "#/$defs/BufferGateConfig"
},
{
"type": "null"
}
],
"description": "Send all kept messages when a condition on a state bucket is true.\nBuckets are global, so a different pipeline can write the value that\nopens the gate."
}
},
"title": "buffer",
"type": "object"
},
"CastType": {
"description": "The type that a `cast` converts a value to.\n\nThese types are not the column types of the database outputs. There is no\n`bigint` and no `decimal`. Here, `json` parses a string that contains JSON.",
"oneOf": [
{
"const": "text",
"description": "A string. A number or a boolean is written as JSON writes it. An object\nor an array is an error.",
"type": "string"
},
{
"const": "integer",
"description": "A whole number. A string is parsed. A number with a fractional part is\nan error. The cast does not round.",
"type": "string"
},
{
"const": "float",
"description": "A number. A string is parsed.",
"type": "string"
},
{
"const": "boolean",
"description": "True or false. The cast accepts the strings `true` and `false` in\nuppercase or lowercase, and the numbers 1 and 0. Other values are an\nerror.",
"type": "string"
},
{
"const": "timestamp",
"description": "A timestamp, written as RFC 3339. A string is parsed and written again\nin one format. A number is read as **seconds** since the epoch, with\nfractions. The column mapping reads a number in the same way.",
"type": "string"
},
{
"const": "date",
"description": "A calendar date, written as `2026-08-10`. A string can be a date or a\nfull timestamp. From a timestamp, the cast uses the date.",
"type": "string"
},
{
"const": "uuid",
"description": "A UUID, in lowercase. The cast accepts only a string in the canonical\nform with hyphens.",
"type": "string"
},
{
"const": "json",
"description": "Parses the JSON that a string contains. The input must be a string. Use\nit for a payload that is encoded inside another payload.",
"type": "string"
}
]
},
"ClickhouseInputConfig": {
"description": "Reads a `ClickHouse` table, view or query at an interval and sends each row\nas a message. It uses the HTTP interface of `ClickHouse`.\n\n`ClickHouse` sends the rows as `JSONEachRow`. A `DateTime` is ISO 8601, an\n`Int64` is a number, and a `Decimal` keeps its digits. The modes, the\nwatermark and the limits of an incremental read are the same as on the\n`postgres` input.",
"properties": {
"columns": {
"description": "The columns to read, in order. Leave it empty to read all columns. For\nan incremental input, the list must include `field`.",
"items": {
"type": "string"
},
"type": "array"
},
"connection": {
"description": "The name of the clickhouse connection to read through. Declare the\nconnection in the connections file.",
"type": "string",
"x-connection": "clickhouse"
},
"interval_secs": {
"description": "The time between two reads, in seconds. The time starts at the end of a\nread. Thus, two reads never overlap. The first read occurs when the\npipeline starts.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"max_batch": {
"description": "The maximum number of rows in one batch. The default is 1. The input\nputs rows that it already read into a batch. It does not wait for more\nrows to fill a batch.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"mode": {
"$ref": "#/$defs/PollMode",
"description": "Whether each read returns all rows (`snapshot`) or only the rows after\nthe last read (`incremental`)."
},
"page_size": {
"description": "The maximum number of rows that one query returns. The default is\n1000. When a page is full, the input reads the next page immediately.\nThe interval starts after a page that is not full. A `snapshot` ignores\nthis field and reads all rows in one query.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"query": {
"description": "A `SELECT` to read in place of a table. The input puts the query in a\nsubquery. It adds the cursor condition, the `ORDER BY` and the page\nlimit outside the subquery. Thus, the query needs no placeholder and no\n`ORDER BY`.\n\nWrite one statement with no semicolon at the end. You can use all SQL\nthat the server accepts in a subquery, for example `WITH`.",
"type": [
"string",
"null"
]
},
"table": {
"description": "The table or view to read, as `name` or `schema.name`. Give exactly one\nof `table` and `query`.",
"type": [
"string",
"null"
]
}
},
"required": [
"connection",
"interval_secs",
"mode"
],
"title": "clickhouse",
"type": "object"
},
"ClickhouseOutputConfig": {
"description": "Inserts each batch into a `ClickHouse` table, one insert for each batch.\n\n`columns` has the same format as on the `postgres` output. Each entry names\na column, its type and the field to read. The default `field` is the name of\nthe column. Without `columns`, the table has a `payload` column that\ncontains each message as JSON text.\n\n`ClickHouse` has no auto-increment column and no unique constraint.\n`order_by` names the sorting key of the `MergeTree` table. If you do not set\nit, the table gets a `received_at` timestamp and is sorted by it. A sorting\nkey does not remove duplicate rows.\n\nThe output creates the table if it does not exist. Set `create_table` to\nfalse for a table that another system owns. The output does not change an\nexisting table.",
"properties": {
"columns": {
"description": "The column for each message field. If you do not set it, the output\nkeeps each full message as JSON text in a `payload` column.",
"items": {
"$ref": "#/$defs/ColumnMapping"
},
"type": "array"
},
"connection": {
"description": "The name of the clickhouse connection to insert through. Declare the\nconnection in the connections file. The connection gives the url, the\ndatabase and the user.",
"type": "string",
"x-connection": "clickhouse"
},
"create_table": {
"description": "Create the table on start if it does not exist. The default is true.",
"type": [
"boolean",
"null"
]
},
"on_extra_fields": {
"$ref": "#/$defs/ExtraFieldPolicy",
"description": "What to do with a message that has fields that no column reads."
},
"order_by": {
"description": "The columns that sort the created table. This is the sorting key of the\n`MergeTree` table and its index. If you do not set it, the table gets a\n`received_at` timestamp and is sorted by it. The output makes these\ncolumns `NOT NULL`, because `ClickHouse` cannot sort by a nullable key.",
"items": {
"type": "string"
},
"type": "array"
},
"table": {
"description": "The table to insert into. The output creates it if it does not exist.\nYou can add a database (`analytics.readings`). This database replaces\nthe database of the connection. Use only letters, digits and\nunderscores.",
"type": "string"
}
},
"required": [
"connection",
"table"
],
"title": "clickhouse",
"type": "object"
},
"ColumnMapping": {
"description": "One message field that the output writes to one column.\n\nThe default of `field` is `name`. When the message uses the column names,\ngive only `name` and `type`. `field` is a dotted path, for example\n`_meta.subject`. A key that contains a dot and matches exactly has\npriority over the path.",
"properties": {
"field": {
"description": "The field to read, as a dotted path. The default is the name of the\ncolumn.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"message": {
"description": "Write the full message to this column. Use it only with a `json`\ncolumn. Do not use it with `field`.",
"type": "boolean"
},
"name": {
"description": "The name of the column in the table. Use only letters, digits and\nunderscores.",
"type": "string"
},
"nullable": {
"description": "Whether the column accepts `NULL`. The default is true. With `false`,\nthe output makes the column `NOT NULL`, and a missing field is an\nerror.",
"type": [
"boolean",
"null"
]
},
"on_missing": {
"anyOf": [
{
"$ref": "#/$defs/MissingColumnPolicy"
},
{
"type": "null"
}
],
"description": "What to do with a message that does not have the field. The default is\n`null`, or `error` for a column that is not nullable."
},
"type": {
"$ref": "#/$defs/ColumnType",
"description": "The type of the column. The output checks each value against it."
}
},
"required": [
"name",
"type"
],
"title": "column",
"type": "object"
},
"ColumnType": {
"description": "The type of a column. The names are the same for all database outputs.\nEach output changes them into the types of its server.\n\nThe output checks each value against the type before it sends the value.\nIt does not convert values. For example, the string `\"12.5\"` in a `float`\ncolumn is an error. To convert a value, use a `cast` in a `map` transform.",
"oneOf": [
{
"const": "text",
"description": "A string of any length. Only a JSON string is accepted.",
"type": "string"
},
{
"const": "integer",
"description": "A 32-bit whole number. A JSON number with a fractional part is an\nerror. A number outside the range is an error.",
"type": "string"
},
{
"const": "bigint",
"description": "A 64-bit whole number.",
"type": "string"
},
{
"const": "float",
"description": "A double-precision floating point number.",
"type": "string"
},
{
"const": "decimal",
"description": "An exact decimal. The output sends the digits as they are in the\nmessage, so no precision is lost.",
"type": "string"
},
{
"const": "boolean",
"description": "True or false. Only a JSON boolean is accepted.",
"type": "string"
},
{
"const": "timestamp",
"description": "A date and time with a time zone. The server parses a JSON string as\nISO 8601 or RFC 3339. The output reads a JSON number as **seconds**\nsince the epoch, with fractions.",
"type": "string"
},
{
"const": "date",
"description": "A calendar date, as a JSON string (`2026-08-10`).",
"type": "string"
},
{
"const": "uuid",
"description": "A UUID, as a JSON string.",
"type": "string"
},
{
"const": "json",
"description": "Any JSON value, stored as JSON.",
"type": "string"
}
]
},
"ConcatPart": {
"description": "One piece of a [`Mapping::Concat`].",
"oneOf": [
{
"description": "A value from the message. A string is used as it is. A number or a\nboolean is written as JSON writes it. An object or an array is an error.",
"properties": {
"field": {
"description": "The field to read.",
"type": "string"
},
"type": {
"const": "field",
"type": "string"
}
},
"required": [
"type",
"field"
],
"type": "object"
},
{
"description": "Literal text, for example a separator, a prefix or a suffix.",
"properties": {
"type": {
"const": "value",
"type": "string"
},
"value": {
"description": "The text.",
"type": "string"
}
},
"required": [
"type",
"value"
],
"type": "object"
}
]
},
"Condition": {
"description": "One test on a message.\n\n`filter`, `remember` and the gate of a `buffer` use conditions. A list of\nconditions means that all of them must match. There is no `or` and no\nnesting. Use `invert` on a `filter`, or `none_of`, for a negative test.",
"oneOf": [
{
"description": "Compares a field to a number. A message whose field is missing or is not\na number does not match.",
"properties": {
"field": {
"description": "The field to test, as a dotted path.",
"type": "string",
"x-message-field": true
},
"operator": {
"$ref": "#/$defs/NumericFilterOperatorKind",
"description": "How to compare the field."
},
"type": {
"const": "numeric",
"type": "string"
},
"value": {
"description": "The number to compare to.",
"format": "double",
"type": "number"
}
},
"required": [
"type",
"field",
"operator",
"value"
],
"type": "object"
},
{
"description": "Compares a field to a string. A message whose field is missing or is not\na string does not match.",
"properties": {
"field": {
"description": "The field to test, as a dotted path.",
"type": "string",
"x-message-field": true
},
"operator": {
"$ref": "#/$defs/StringFilterOperatorKind",
"description": "How to compare the field."
},
"type": {
"const": "string",
"type": "string"
},
"value": {
"description": "The string to compare to.",
"type": "string"
}
},
"required": [
"type",
"field",
"operator",
"value"
],
"type": "object"
},
{
"description": "Matches when the field is a string equal to one of `values`. A message\nwhose field is missing or is not a string does not match.",
"properties": {
"field": {
"description": "The field to test, as a dotted path.",
"type": "string",
"x-message-field": true
},
"type": {
"const": "one_of",
"type": "string"
},
"values": {
"description": "The strings that match.",
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"type",
"field",
"values"
],
"type": "object"
},
{
"description": "Matches when the field is a string equal to none of `values`. A message\nwhose field is missing or is not a string also does not match.",
"properties": {
"field": {
"description": "The field to test, as a dotted path.",
"type": "string",
"x-message-field": true
},
"type": {
"const": "none_of",
"type": "string"
},
"values": {
"description": "The strings that do not match.",
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"type",
"field",
"values"
],
"type": "object"
}
]
},
"DeadbandMode": {
"description": "The unit of the `delta` of a deadband: an amount, or a percentage of the\nlast value that passed.",
"oneOf": [
{
"const": "absolute",
"description": "`delta` is in the units of the field. For example, `0.5` is half a\ndegree.",
"type": "string"
},
{
"const": "percent",
"description": "`delta` is a percentage of the last value that passed. For example,\n`2` is two percent. When the last value is zero, every message passes.",
"type": "string"
}
]
},
"DeadbandTransformConfig": {
"description": "Drops a message when its field did not change sufficiently since the last\nmessage that passed. This is a filter that keeps state per key.\n\nThe first message for each key always passes. After that, a message passes\nin two cases:\n\n- `field` differs from the last value that passed by more than `delta`.\n- `max_seconds` went by since the last message passed.\n\nWith `flatline_seconds`, the transform also finds a stuck sensor. When the\nvalue does not change for that time, the next message passes with\n`stuck: true`. This occurs one time for each flat period.",
"properties": {
"delta": {
"description": "The change that a message needs to pass. The unit is set by `mode`.",
"format": "double",
"type": "number"
},
"field": {
"description": "The numeric field to compare.",
"type": "string",
"x-message-field": true
},
"flatline_seconds": {
"description": "After this number of seconds with no change, pass the next message with\n`stuck: true`. This occurs one time for each flat period.",
"format": "double",
"type": [
"number",
"null"
]
},
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out for\none series.",
"items": {
"type": "string"
},
"type": "array"
},
"max_seconds": {
"description": "Pass a message when this number of seconds went by since the last\nmessage passed, also if the value did not change.",
"format": "double",
"type": [
"number",
"null"
]
},
"mode": {
"$ref": "#/$defs/DeadbandMode",
"description": "The unit of `delta`. The default is `absolute`."
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have the field or a group\nfield. The default is `error`."
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"time": {
"description": "The field that holds the time of each message, as an RFC 3339 string\nor as milliseconds since the epoch. Leave it out to use the arrival\ntime.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"field",
"delta"
],
"title": "deadband",
"type": "object"
},
"Derivation": {
"description": "One derived value and the field it is written to.",
"properties": {
"as": {
"description": "The field to write the result to.",
"type": "string"
},
"field": {
"description": "The numeric field to read.",
"type": "string",
"x-message-field": true
},
"function": {
"$ref": "#/$defs/DeriveFnKind",
"description": "The calculation to do."
},
"wrap_at": {
"description": "For `counter`: the value at which the counter goes back to zero. With\nit, the transform reads a decrease as a wrap. Without it, the\ntransform reads a decrease as a reset.",
"format": "double",
"type": [
"number",
"null"
]
}
},
"required": [
"function",
"field",
"as"
],
"title": "derivation",
"type": "object"
},
"DeriveFnKind": {
"description": "How the transform calculates a value from this message and the previous\nmessage.",
"oneOf": [
{
"const": "rate",
"description": "The change per second since the previous message, from the `time`\nfield. The value is `null` for the first message and when no time went\nby.",
"type": "string"
},
{
"const": "delta",
"description": "The change since the previous message. The value is `null` for the\nfirst message.",
"type": "string"
},
{
"const": "cumsum",
"description": "The running total of the field, from the first message.",
"type": "string"
},
{
"const": "counter",
"description": "The running total of the increases. Use it for a counter that resets\nor wraps. When the value decreases and `wrap_at` is set, the transform\nadds the increase through `wrap_at`. When `wrap_at` is not set, it adds\nthe new value.",
"type": "string"
}
]
},
"DeriveTransformConfig": {
"description": "Writes a value onto each message that needs the previous message of the\nsame key. The value can be a rate of change, a delta, a running total or a\ncounter that wraps.\n\nYou can give many derivations. The transform writes each one to its own\n`as` field. The first message for each key has no previous message. For\nthat message, `rate` and `delta` write `null`.",
"properties": {
"derive": {
"description": "The values to calculate. Give one or more, each with a different\n`as`.",
"items": {
"$ref": "#/$defs/Derivation"
},
"type": "array"
},
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out for\none series.",
"items": {
"type": "string"
},
"type": "array"
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have a derived field or a\ngroup field. The default is `error`."
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"time": {
"description": "The field that holds the time of each message, as an RFC 3339 string\nor as milliseconds since the epoch. Leave it out to use the arrival\ntime.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"derive"
],
"title": "derive",
"type": "object"
},
"DetectLearn": {
"description": "The readings that a `detect` baseline learns from, for the methods that\ncontinue to learn.",
"oneOf": [
{
"const": "all",
"description": "All readings, with anomalies included. A change that continues becomes\nthe new normal at the speed of the baseline.",
"type": "string"
},
{
"const": "normal_only",
"description": "Only the readings that are not anomalies. An anomaly cannot change the\nbaseline. Use it with `readapt_after_seconds`. Without it, the\ntransform flags a real change of level permanently.",
"type": "string"
}
]
},
"DetectMethod": {
"description": "How the transform finds an anomaly.\n\nThe window methods (`zscore`, `mad`) compare a value with the values before\nit. The window does not include the value, so a spike does not change its\nown baseline. The chart methods (`cusum` without `target`, `ewma_chart`,\n`western_electric`) fix their baseline at the end of the warm-up.",
"oneOf": [
{
"description": "An anomaly is more than `threshold` standard deviations from the mean\nof the last `size` values.",
"properties": {
"size": {
"description": "The number of earlier values in the baseline.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"threshold": {
"description": "The number of standard deviations that makes an anomaly. The\ndefault is `3`.",
"format": "double",
"type": [
"number",
"null"
]
},
"type": {
"const": "zscore",
"type": "string"
}
},
"required": [
"type",
"size"
],
"type": "object"
},
{
"description": "An anomaly is more than `threshold` scaled median absolute deviations\nfrom the median of the last `size` values. Use it when the baseline\ncontains outliers.",
"properties": {
"size": {
"description": "The number of earlier values in the baseline.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"threshold": {
"description": "The number of scaled MADs that makes an anomaly. The default is\n`3.5`.",
"format": "double",
"type": [
"number",
"null"
]
},
"type": {
"const": "mad",
"type": "string"
}
},
"required": [
"type",
"size"
],
"type": "object"
},
{
"description": "A two-sided CUSUM. It adds the drift above `target` and the drift below\n`target` in two sums. An anomaly is a sum that is more than\n`threshold`. Then that sum goes back to zero. Use it to find a small\nshift that continues.",
"properties": {
"drift": {
"description": "The change per message that does not count as drift, in the units\nof the field.",
"format": "double",
"type": "number"
},
"target": {
"description": "The expected value of the series. Leave it out to use the mean of\nthe warm-up.",
"format": "double",
"type": [
"number",
"null"
]
},
"threshold": {
"description": "The sum of drift that makes an anomaly.",
"format": "double",
"type": "number"
},
"type": {
"const": "cusum",
"type": "string"
}
},
"required": [
"type",
"drift",
"threshold"
],
"type": "object"
},
{
"description": "An EWMA control chart. An anomaly is a smoothed value outside a band of\n`threshold` standard deviations around the mean of the warm-up. It\nfinds small shifts and ignores single points.",
"properties": {
"alpha": {
"description": "The weight of the newest value, from 0 to 1. The default is `0.2`.",
"format": "double",
"type": [
"number",
"null"
]
},
"threshold": {
"description": "The width of the band, in standard deviations. The default is `3`.",
"format": "double",
"type": [
"number",
"null"
]
},
"type": {
"const": "ewma_chart",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"description": "The Western Electric rules, against the mean and deviation of the\nwarm-up:\n\n- one point more than 3σ from the mean,\n- two of three points more than 2σ from the mean on one side,\n- four of five points more than 1σ from the mean on one side,\n- eight points in a sequence on one side.\n\nThe transform writes the rule that matched beside the flag.",
"properties": {
"type": {
"const": "western_electric",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"description": "An anomaly is `size` identical values in a sequence. Use it to find a\nstuck instrument.",
"properties": {
"size": {
"description": "The number of identical values in a sequence that makes an anomaly.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"type": {
"const": "flatline",
"type": "string"
}
},
"required": [
"type",
"size"
],
"type": "object"
},
{
"description": "An anomaly is more than `threshold` deviations from a baseline that\ncontinues to learn. The baseline is an exponentially weighted mean and\nspread, each with its own time constant. Use it for a series that\ndrifts slowly and arrives at irregular times. It reads the `time` field\nof the transform.",
"properties": {
"mean_tau_seconds": {
"description": "The time constant of the mean, in seconds. A smaller value makes the\nmean follow the series more quickly.",
"format": "double",
"type": "number"
},
"min_spread": {
"description": "The minimum deviation, in the units of the field. Without it, a\nsignal that was very quiet flags its first small change. The default\nis `0`.",
"format": "double",
"type": [
"number",
"null"
]
},
"spread_tau_seconds": {
"description": "The time constant of the spread, in seconds. Make it longer than\n`mean_tau_seconds`, so that a short period of noise does not make\nthe band wider immediately.",
"format": "double",
"type": "number"
},
"threshold": {
"description": "The number of deviations that makes an anomaly. The default is `3`.",
"format": "double",
"type": [
"number",
"null"
]
},
"type": {
"const": "ewma",
"type": "string"
}
},
"required": [
"type",
"mean_tau_seconds",
"spread_tau_seconds"
],
"type": "object"
}
]
},
"DetectMode": {
"description": "Which messages the `detect` transform sends.",
"oneOf": [
{
"const": "annotate",
"description": "Every message passes, with the flag and the score.",
"type": "string"
},
{
"const": "only_anomalies",
"description": "Only the anomalies pass, with the flag and the score.",
"type": "string"
}
]
},
"DetectTransformConfig": {
"description": "Flags anomalies in a numeric field against its own series. `method` selects\nhow the transform finds an anomaly.\n\nThe transform writes a boolean under `as`. The default is `anomaly`. It also\nwrites a score under `<as>_score`. The score is the distance from normal, in\nthe units of the method. A `filter` after the transform can use a stricter\nlimit on the score.\n\nThe warm-up is `min_samples` messages for each key. The transform flags\nnothing during the warm-up.",
"properties": {
"as": {
"description": "The field to write the flag to. The score goes to `<as>_score`. The\ndefault is `anomaly`.",
"type": [
"string",
"null"
]
},
"field": {
"description": "The numeric field to examine.",
"type": "string",
"x-message-field": true
},
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out for\none series.",
"items": {
"type": "string"
},
"type": "array"
},
"learn": {
"$ref": "#/$defs/DetectLearn",
"description": "For `zscore`, `mad` and `ewma`: the readings that the baseline learns\nfrom. The default is `all`."
},
"method": {
"$ref": "#/$defs/DetectMethod",
"description": "How the transform finds an anomaly."
},
"min_samples": {
"description": "The number of messages for each key before the transform flags\nanything. The default is the `size` of the method, or 30 for a method\nwith no `size`.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"mode": {
"$ref": "#/$defs/DetectMode",
"description": "Which messages to send. The default is `annotate`."
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have the field or a group\nfield. The default is `error`."
},
"readapt_after_seconds": {
"description": "With `learn: normal_only`: when readings are anomalies for this number\nof seconds without a break, the baseline learns from them. A change\nthat continues then becomes the new normal.",
"format": "double",
"type": [
"number",
"null"
]
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"time": {
"description": "The field that holds the time of each message, as an RFC 3339 string\nor as milliseconds since the epoch. The `ewma` method and\n`readapt_after_seconds` use it. Leave it out to use the arrival time.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"with_baseline": {
"description": "Also write the baseline of the reading, in the units of the field. The\nnormal value goes to `<as>_expected`. The permitted distance from it\ngoes to `<as>_band`. The value is `null` when the method has no\nbaseline.",
"type": "boolean"
}
},
"required": [
"field",
"method"
],
"title": "detect",
"type": "object"
},
"DummyConfig": {
"description": "Sends one generated message at a fixed interval. Use it to test a pipeline\nwithout a real source.\n\nEach message contains a `value` and the `current_time` at which the input\nsent it. The `payload` field sets the type of `value`. It can be a number\nfrom a sine wave or a random sentence.",
"properties": {
"amplitude": {
"description": "The peak of the sine wave. The value goes from `-amplitude` to\n`+amplitude`. Applies only to the `number` payload. The default is 1.",
"format": "double",
"type": [
"number",
"null"
]
},
"duration": {
"description": "The time between two messages, in s.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"payload": {
"anyOf": [
{
"$ref": "#/$defs/DummyPayload"
},
{
"type": "null"
}
],
"description": "The type of `value` in each message. `number` is a number from a sine\nwave. `text` is a random sentence. The default is `number`."
},
"period": {
"description": "The time for one full cycle of the sine wave, in s. Applies only to the\n`number` payload. The default is 60. The input samples the wave by the\nclock, so the period does not change with `duration`.",
"format": "double",
"type": [
"number",
"null"
]
}
},
"required": [
"duration"
],
"title": "dummy",
"type": "object"
},
"DummyPayload": {
"description": "The type of `value` in each message from a `dummy` input.",
"oneOf": [
{
"const": "number",
"description": "A number from a sine wave.",
"type": "string"
},
{
"const": "text",
"description": "A random sentence.",
"type": "string"
}
]
},
"EnvelopeConfig": {
"description": "How an input adds metadata about the source of each message.\n\nThe \"metadata\" section of each input lists its metadata. Examples are the\nsubject of a nats message and the topic, partition and offset of a kafka\nrecord. The metadata also contains the pipeline and the input type.\n\nThe input adds the metadata as ordinary fields on the message. Thus each\ntransform can use it as it uses the fields of the payload, for example\n`\"group_by\": [\"_meta.subject\"]`.\n\nIf you do not set an envelope, the input sends each message without\nchanges. An envelope changes the shape of each message from the input.\nUpdate the field paths downstream when you add one.",
"oneOf": [
{
"description": "Add the metadata as one more field on the message. The fields of the\npayload do not move.\n\nThis type works only on a payload that is a JSON object. The input\nskips a message that is a number or a string and writes a warning to\nthe log. Use `wrap` for these payloads.",
"properties": {
"meta": {
"description": "The field for the metadata object. The default is `_meta`.",
"type": [
"string",
"null"
]
},
"type": {
"const": "merge",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"description": "Put the full payload under a field, next to the metadata:\n`{\"value\": …, \"_meta\": {…}}`.\n\nThis type works with all payloads, for example a `1` or a `\"recipe-a\"`.\nEach field path downstream must then start with the payload field, for\nexample `value.temperature`.",
"properties": {
"meta": {
"description": "The field for the metadata object. The default is `_meta`.",
"type": [
"string",
"null"
]
},
"payload": {
"description": "The field for the original payload. The default is `value`.",
"type": [
"string",
"null"
]
},
"type": {
"const": "wrap",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
}
]
},
"ExtraFieldPolicy": {
"description": "What to do with a message that has fields that no column reads.",
"oneOf": [
{
"const": "ignore",
"description": "Write the mapped columns and ignore the other fields. This is the\ndefault.",
"type": "string"
},
{
"const": "error",
"description": "Fail the batch. Use it for a stream with a fixed shape, where a new\nfield is a problem.",
"type": "string"
}
]
},
"FeatureKind": {
"description": "A number that describes a window of readings.",
"oneOf": [
{
"const": "mean",
"description": "The arithmetic mean.",
"type": "string"
},
{
"const": "std",
"description": "The population standard deviation.",
"type": "string"
},
{
"const": "min",
"description": "The smallest value.",
"type": "string"
},
{
"const": "max",
"description": "The largest value.",
"type": "string"
},
{
"const": "range",
"description": "The largest value minus the smallest value.",
"type": "string"
},
{
"const": "slope",
"description": "The least-squares slope. It is per second with the `time` field, and\nper message without it.",
"type": "string"
},
{
"const": "skew",
"description": "The skewness: the direction of the longer tail.",
"type": "string"
},
{
"const": "kurtosis",
"description": "The excess kurtosis: the weight of the tails.",
"type": "string"
},
{
"const": "rms",
"description": "The root mean square.",
"type": "string"
},
{
"const": "crest_factor",
"description": "The peak magnitude divided by the RMS.",
"type": "string"
},
{
"const": "zero_crossings",
"description": "The number of times that the signal crosses zero.",
"type": "string"
},
{
"const": "n_peaks",
"description": "The number of local maxima.",
"type": "string"
},
{
"const": "autocorr1",
"description": "The autocorrelation at lag one.",
"type": "string"
},
{
"const": "dominant_frequency",
"description": "The strongest frequency above DC, in hertz. It needs a sample rate from\nthe `time` field or from `sample_rate_hz`.",
"type": "string"
},
{
"const": "count",
"description": "The number of readings in the window.",
"type": "string"
},
{
"const": "duration",
"description": "The time from the first reading to the last reading, in seconds, from\nthe `time` field.",
"type": "string"
}
]
},
"FeaturesTransformConfig": {
"description": "Changes a batch of readings into one message of features for each group.\nUse it to send a small set of numbers to a model, in place of the raw\nreadings. Put a `buffer` on the input. Without it, each batch has only one\nreading.\n\nThe transform writes:\n\n- each feature in `include` under its own name, for example `mean`, `rms`\n or `crest_factor`,\n- each entry in `bands` under its `as`,\n- the `group_by` fields under their leaf names, as `reduce` does.\n\nA feature with no value for the window is `null`. For example, a slope of\none point is `null`. The spectral features (`dominant_frequency`, `bands`)\nneed a sample rate. kayak uses `sample_rate_hz`, or calculates the rate\nfrom the `time` field. Without one of the two, the pipeline does not build.\nThis transform keeps no state and needs no state bucket.",
"properties": {
"bands": {
"description": "The frequency bands to calculate the power of. The transform writes\neach one under its `as`. Give `include`, `bands` or both.",
"items": {
"$ref": "#/$defs/Band"
},
"type": "array"
},
"field": {
"description": "The numeric field to read.",
"type": "string",
"x-message-field": true
},
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out to use\nthe full batch as one window.",
"items": {
"type": "string"
},
"type": "array"
},
"include": {
"description": "The features to calculate. The transform writes each one under its own\nname.",
"items": {
"$ref": "#/$defs/FeatureKind"
},
"type": "array"
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a reading that does not have the field or a group\nfield. The default is `error`."
},
"sample_rate_hz": {
"description": "The sample rate of the readings in hertz, for the spectral features.\nIt must be more than zero. kayak uses it in place of the rate from\n`time`. Use it when the source has timestamps with low resolution.",
"format": "double",
"type": [
"number",
"null"
]
},
"time": {
"description": "The field that holds the time of each reading, as an RFC 3339 string or\nas milliseconds since the epoch. `slope` and `duration` use it for\nseconds. The spectral features use it for the sample rate.",
"type": [
"string",
"null"
],
"x-message-field": true
}
},
"required": [
"field"
],
"title": "features",
"type": "object"
},
"FileFormat": {
"description": "The layout of the messages in a file.\n\nBoth formats are JSON. Use `ndjson` for a stream. An `ndjson` file is valid\nafter each batch, so you can read it while the pipeline runs or after a\ncrash.",
"oneOf": [
{
"const": "ndjson",
"description": "One JSON message on each line. The output adds each message when it\narrives.",
"type": "string"
},
{
"const": "json_array",
"description": "The file is one JSON array. The output closes the array when the file\nrotates.",
"type": "string"
}
]
},
"FileOutputConfig": {
"description": "Writes each batch to files in a directory on the server.\n\nA `file` connection gives the root directory, and `path` is relative to it.\nThe root must be inside the `--data-dir` of the server. Without that flag,\na `file` output fails to build. The output names each file\n`<open time>-<sequence>.<ext>`, so the names sort by time and are unique.\n\nUse the `file` output for local development and tests.",
"properties": {
"connection": {
"description": "The name of the file connection to write under. Declare the connection\nin the connections file. The connection gives the root directory.",
"type": "string",
"x-connection": "file"
},
"format": {
"anyOf": [
{
"$ref": "#/$defs/FileFormat"
},
{
"type": "null"
}
],
"description": "The layout of the messages. The default is `ndjson`."
},
"path": {
"description": "The directory to write into, relative to the root of the connection,\nfor example `orders`. The path must stay inside the root. kayak refuses\nan absolute path and a path that contains `..`.",
"type": "string"
},
"rotate": {
"anyOf": [
{
"$ref": "#/$defs/RotationConfig"
},
{
"type": "null"
}
],
"description": "When to close a file and start the next file. Without this setting, the\noutput writes one file while the pipeline runs."
}
},
"required": [
"connection",
"path"
],
"title": "file",
"type": "object"
},
"FilterTransformConfig": {
"description": "Keeps the messages that match all `conditions` and drops the other\nmessages. With `invert`, it drops the messages that match and keeps the\nother messages.\n\nThe transform drops a batch that has no messages left. A message that does\nnot have the field of a condition does not match that condition. A message\nwith a field of the wrong type also does not match. Thus `invert` keeps\nthese messages.",
"properties": {
"conditions": {
"description": "The conditions that a message must match. At least one condition is\nrequired.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"invert": {
"description": "Drop the messages that match, and keep the other messages. The default\nis false.",
"type": "boolean"
}
},
"required": [
"conditions"
],
"title": "filter",
"type": "object"
},
"HttpAuthConfig": {
"description": "A credential in a header. The `http` input checks it on each post. The\n`http` output, the `http` transform and the `http_poll` input send it on\neach request.\n\nThis credential is for one pipeline only. It is not related to the user\naccounts of the server.\n\nThe sender sends the same token on each request. Thus the token is only as\nsecure as the connection. kayak serves plain HTTP. Put TLS in front of\nkayak, or other systems on the network path can read the token.",
"oneOf": [
{
"description": "A token in the standard `Authorization` header, as\n`Authorization: Bearer TOKEN`. Use this variant if the other system can\nuse that header.",
"properties": {
"token": {
"$ref": "#/$defs/Secret",
"description": "The token. Use a `${NAME}` reference, so that the config file keeps\nonly the name and the secret store keeps the value."
},
"type": {
"const": "bearer",
"type": "string"
}
},
"required": [
"type",
"token"
],
"type": "object"
},
{
"description": "A fixed value in a header that you name. Use this variant for a system\nthat cannot use the `Authorization` header.",
"properties": {
"name": {
"description": "The name of the header. The `http` input compares the name without\ncase. On an `http` input, the name must not be a header that an\n`envelope` copies into the messages.",
"type": "string"
},
"type": {
"const": "header",
"type": "string"
},
"value": {
"$ref": "#/$defs/Secret",
"description": "The exact value of the header. Use a `${NAME}` reference."
}
},
"required": [
"type",
"name",
"value"
],
"type": "object"
}
]
},
"HttpBodyKind": {
"description": "The body of one request from an `http` output or an `http` transform.\n\nSelect the value that the API at the endpoint expects. Use `batch` for an\nendpoint that takes an array. Use `message` for a webhook that takes one\nevent for each call.",
"oneOf": [
{
"const": "batch",
"description": "The full batch as one JSON array, in one request. The default.",
"type": "string"
},
{
"const": "message",
"description": "One request for each message. The body is the message. The requests go\nin sequence. The first failure fails the batch, and kayak does not send\nthe messages after it.",
"type": "string"
}
]
},
"HttpInputConfig": {
"description": "Accepts messages that are posted to the endpoint of the pipeline,\n`POST /api/pipelines/{id}/messages`.\n\nkayak makes the endpoint from the pipeline id. The endpoint is available\nwhen the pipeline runs. The body is one JSON message or an array of\nmessages. An array becomes one batch. A pipeline can have only one `http`\ninput. A second `http` input fails to build.",
"properties": {
"auth": {
"anyOf": [
{
"$ref": "#/$defs/HttpAuthConfig"
},
{
"type": "null"
}
],
"description": "The credential that a post must have. If you do not set it, the\nendpoint accepts all posts. A post without the correct credential gets\n`401`."
},
"capacity": {
"description": "The maximum number of posted batches in the queue before the pipeline.\nThe default is 1024. When the queue is full, the endpoint refuses a\npost with `503`.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
}
},
"title": "http",
"type": "object"
},
"HttpOutputConfig": {
"description": "Sends the batch to an http endpoint, for example a webhook or an ingest\nAPI.\n\nThe output ignores the body of the reply. A status other than 2xx fails the\nbatch. The error contains the reply of the endpoint. Use the `http`\ntransform if the pipeline needs the reply.",
"properties": {
"auth": {
"anyOf": [
{
"$ref": "#/$defs/HttpAuthConfig"
},
{
"type": "null"
}
],
"description": "The credential that the output sends. If you do not set it, the output\nsends no credential."
},
"body": {
"anyOf": [
{
"$ref": "#/$defs/HttpBodyKind"
},
{
"type": "null"
}
],
"description": "The content of one request. The default is `batch`, one request for\neach batch."
},
"timeout_seconds": {
"description": "The maximum time for one request, in s. The default is 30. A request\nthat times out fails the batch. Thus a slow endpoint stops the pipeline\nfor this time at most.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"url": {
"description": "The endpoint to send to, for example\n`https://example.com/hooks/readings`.",
"type": "string"
},
"verb": {
"anyOf": [
{
"$ref": "#/$defs/HttpVerb"
},
{
"type": "null"
}
],
"description": "The http method. The default is `POST`. `GET` and `DELETE` fail to\nbuild, because a request with no body cannot send the messages."
}
},
"required": [
"url"
],
"title": "http",
"type": "object"
},
"HttpPollConfig": {
"description": "Gets a url at an interval and sends the full reply each time.\n\nEach read is a `GET`. A reply that is an array gives one message for each\nelement. Any other reply gives one message. Use `items` for a reply that\nhas the records inside it, for example `{\"data\": {\"machines\": [...]}}`.\n\nUse this input for reference data, for example a list of machines or\nrecipes that changes rarely. The input has no watermark and no pages. Each\nread sends all records again. Send them to an output that writes the latest\nvalue for each key.\n\nA read fails when the url cannot be reached, when the status is not 2xx,\nwhen the body is not JSON, or when `items` finds nothing. The input reports\nthe failure one time and tries again with backoff. The interval starts\nagain after the next successful read.",
"properties": {
"auth": {
"anyOf": [
{
"$ref": "#/$defs/HttpAuthConfig"
},
{
"type": "null"
}
],
"description": "The credential that the input sends. If you do not set it, the input\nsends no credential."
},
"interval_secs": {
"description": "The time between two reads, in s. The time starts at the end of one\nread. The first read occurs when the pipeline starts.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"items": {
"description": "The position of the records in the reply, as a JSON pointer. For\nexample, `/data/machines` reads the array at `data.machines`. If you do\nnot set it, the input uses the full reply. An array gives one message\nfor each element. Any other value gives one message.",
"type": [
"string",
"null"
]
},
"max_batch": {
"description": "The maximum number of messages in one batch. The default is 1. The input\nputs only messages that are already read into a batch. It does not wait\nfor a batch to fill.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"timeout_seconds": {
"description": "The maximum time for one request, in s. The default is 30.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"url": {
"description": "The url to get, for example `https://erp.example.com/api/machines`.",
"type": "string"
}
},
"required": [
"url",
"interval_secs"
],
"title": "http_poll",
"type": "object"
},
"HttpResponseKind": {
"description": "What an `http` transform does with the reply.",
"oneOf": [
{
"const": "replace",
"description": "The reply becomes the new batch. With `body: batch`, the reply must be a\nJSON array of messages. With `body: message`, the reply is a message or\nan array of messages. The default.",
"type": "string"
},
{
"const": "merge",
"description": "The transform writes the reply onto the message that caused it, under\n`as`. With `body: batch`, an array reply with one entry for each message\ngoes to the messages in sequence. Any other reply goes onto each\nmessage. The messages keep all their fields.",
"type": "string"
}
]
},
"HttpTransformConfig": {
"description": "Sends the batch to an http endpoint and continues with the reply. Use it\nto call a model or another service.\n\n`body` sets the content of one request: the full batch as a JSON array, or\none message. `wrap` puts the body under a key, for example\n`{\"instances\": …}`. `response` sets what the transform does with the reply.\n`replace` makes the reply the new batch. `merge` writes the reply onto the\nmessage under `as`. `unwrap` reads the reply from under a key first.\n\nA status other than 2xx fails the batch. The error contains the reply of the\nendpoint. A network failure, a 5xx or a 429 is tried again `retries` times\nwith backoff.\n\nFor a model, put a `buffer` and a `features` transform before this\ntransform. Use `response: merge` to keep the identifiers of the message.",
"properties": {
"as": {
"description": "The field to write the reply under. Applies to `response: merge`.",
"type": [
"string",
"null"
]
},
"auth": {
"anyOf": [
{
"$ref": "#/$defs/HttpAuthConfig"
},
{
"type": "null"
}
],
"description": "The credential that the transform sends. If you do not set it, the\ntransform sends no credential."
},
"body": {
"anyOf": [
{
"$ref": "#/$defs/HttpBodyKind"
},
{
"type": "null"
}
],
"description": "The content of one request. The default is `batch`."
},
"response": {
"anyOf": [
{
"$ref": "#/$defs/HttpResponseKind"
},
{
"type": "null"
}
],
"description": "What to do with the reply. The default is `replace`."
},
"retries": {
"description": "The number of times to send a request again before the batch fails.\nApplies to a network failure, a 5xx and a 429. The default is 0. Each\ntry waits longer than the previous try, and the pipeline waits too.",
"format": "uint32",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"timeout_seconds": {
"description": "The maximum time for one request, in s. The default is 30.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"unwrap": {
"description": "A key to read the reply from, for an API that replies with\n`{\"predictions\": …}`.",
"type": [
"string",
"null"
]
},
"url": {
"description": "The endpoint to send to.",
"type": "string"
},
"verb": {
"$ref": "#/$defs/HttpVerb",
"description": "The http method. `GET` and `DELETE` fail to build, because a request\nwith no body cannot send the messages."
},
"wrap": {
"description": "A key to put the body under, for an API that expects `{\"key\": …}`.",
"type": [
"string",
"null"
]
}
},
"required": [
"url",
"verb"
],
"title": "http",
"type": "object"
},
"HttpVerb": {
"description": "The http method of a request.",
"enum": [
"GET",
"POST",
"PUT",
"PATCH",
"DELETE"
],
"type": "string"
},
"InduInputConfig": {
"description": "Reads live sensors and streams from Indu Cloud through `/api/v1/live/sse`,\nwith the API key of the connection.\n\nName sensors and streams with the ids that the platform uses. Do not use\nUUIDs. The input finds the names through `/api/v1` on the first read. If\nthe key cannot find or see a name, the input reports an error. It then\ntries again after a pause.\n\nEach reading is one message, for example\n`{\"kind\": \"sensor\", \"name\": \"press-3/temperature\", \"value\": 71.2, \"at\": …}`.\nThe message also contains the ids of the platform. When the connection\ndrops, the input connects again with backoff. If the input cannot read\nall readings, it reports an error.",
"properties": {
"backfill": {
"description": "Send the latest value of each series before the live readings. The\ndefault is true. Thus a restarted pipeline has a value for each series\nimmediately.",
"type": [
"boolean",
"null"
]
},
"connection": {
"description": "The name of the indu connection to read through. Declare the connection\nin the connections file.",
"type": "string",
"x-connection": "indu"
},
"max_batch": {
"description": "The maximum number of readings in one batch. The default is 1. The input\nputs only readings that are already received into a batch. It does not\nwait for more readings.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"sensors": {
"description": "The sensors to read, as `<device>/<sensor>`, for example\n`press-3/temperature`. Use the ids that the platform uses. kayak divides\nthe name at the first `/`.",
"items": {
"type": "string"
},
"type": "array"
},
"streams": {
"description": "The streams to read, by the name they were written under, for example\n`press-3/oee`. For a stream that the platform calculates, use its\ndisplay name.",
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"connection"
],
"title": "indu",
"type": "object"
},
"InduOutputConfig": {
"description": "Writes messages into Indu Cloud as streams through\n`POST /ingest/v1/streams`. A stream is a series that is not a sensor.\n\nEach message gives one reading for each entry in `series`. For example, a\nreducer that sends `{machine, oee, availability}` with two series entries\nwrites two streams for each machine. Indu creates an unknown stream when\nthe key of the connection has permission to create streams. If Indu does\nnot accept all rows, the batch fails. The error contains the row errors\nfrom Indu.",
"properties": {
"at": {
"description": "The field that contains the time of the reading, as an RFC 3339 string\nor as ms since the epoch. If you do not set it, the output uses the time\nat which it sends the batch. An `envelope` puts the receive time of an\ninput at `_meta.received_at`.",
"type": [
"string",
"null"
]
},
"connection": {
"description": "The name of the indu connection to write through. Declare the\nconnection in the connections file.",
"type": "string",
"x-connection": "indu"
},
"series": {
"description": "The streams to write, one reading for each message. At least one entry\nis required.",
"items": {
"$ref": "#/$defs/InduSeries"
},
"type": "array"
},
"timeout_seconds": {
"description": "The maximum time for one request, in s. The default is 30.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
}
},
"required": [
"connection",
"series"
],
"title": "indu",
"type": "object"
},
"InduSeries": {
"description": "One series that an `indu` output writes: the stream that gets the value,\nand the field that contains the value.",
"properties": {
"stream": {
"description": "The name of the stream in Indu, for example `press-3/oee`. The name can\ncontain `{field}` placeholders that the output fills from the message,\nfor example `{machine}/oee`. Thus one output can write a stream for each\nmachine. The output skips this series for a message that does not have\nthe field of a placeholder.",
"type": "string"
},
"unit": {
"description": "The unit that Indu records when it creates the stream, for example `%`.\nIndu ignores it when the stream exists.",
"type": [
"string",
"null"
]
},
"value": {
"description": "The field that contains the value, as a path (`oee`, `stats.mean`). The\nvalue must be a number. The output skips this series for a message\nwhere the value is missing or is not a number. The batch does not fail.",
"type": "string"
}
},
"required": [
"stream",
"value"
],
"type": "object"
},
"InputConfig": {
"oneOf": [
{
"$ref": "#/$defs/DummyConfig",
"properties": {
"type": {
"const": "dummy",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/HttpInputConfig",
"properties": {
"type": {
"const": "http",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/KafkaConfig",
"properties": {
"type": {
"const": "kafka",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/NatsConfig",
"properties": {
"type": {
"const": "nats",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/PipelineConfig",
"properties": {
"type": {
"const": "pipeline",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/MqttConfig",
"properties": {
"type": {
"const": "mqtt",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/RedisConfig",
"properties": {
"type": {
"const": "redis",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/OpcuaConfig",
"properties": {
"type": {
"const": "opcua",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/InduInputConfig",
"properties": {
"type": {
"const": "indu",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/PostgresInputConfig",
"properties": {
"type": {
"const": "postgres",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/ClickhouseInputConfig",
"properties": {
"type": {
"const": "clickhouse",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/HttpPollConfig",
"properties": {
"type": {
"const": "http_poll",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
}
],
"properties": {
"ack": {
"anyOf": [
{
"$ref": "#/$defs/AckMode"
},
{
"type": "null"
}
],
"description": "When the input acknowledges a message to its broker. The default is\n`on_receipt`. Only the `kafka` and `mqtt` inputs support `on_delivery`.\nThe `mqtt` input requires a `qos` of `at_least_once` or higher for it.\nOn all other inputs, `on_delivery` fails to build."
},
"buffer": {
"anyOf": [
{
"$ref": "#/$defs/BufferConfig"
},
{
"type": "null"
}
],
"description": "Collect messages from this input into batches before the transforms.\nUse a count (`static`), a time (`tumbling`) or the first of the two\n(`batch`). The buffer never sends an empty batch. Available on all\ninput types. This is not the `buffer` transform."
},
"envelope": {
"anyOf": [
{
"$ref": "#/$defs/EnvelopeConfig"
},
{
"type": "null"
}
],
"description": "Add metadata about the source of each message, for example the subject,\nthe topic or the partition. The \"metadata\" section lists the fields.\nAvailable on all input types. If you do not set it, the input sends each\nmessage without changes."
}
},
"type": "object"
},
"KafkaConfig": {
"description": "Consumes JSON messages from a kafka topic.\n\nThe input skips a payload that is not JSON and writes a warning to the log.\nThe consumer connects on the first read and joins a consumer group. Kafka\nkeeps the read position of the group between restarts.",
"properties": {
"connection": {
"description": "The name of the kafka connection to consume from. Declare the\nconnection in the connections file.",
"type": "string",
"x-connection": "kafka"
},
"group": {
"description": "The consumer group id. Kafka keeps one read position for each group.\nTwo pipelines in the same group divide the topic between them. Two\npipelines in different groups each get all messages.",
"type": "string"
},
"max_batch": {
"description": "The maximum number of messages in one batch. The default is 1.\n\nThe input puts only records that are already received into a batch. It\ndoes not wait for more records, so a high value does not add latency on\na quiet topic. Increase it when the consumer reads a backlog. Each batch\nhas a fixed cost in the pipeline and in each downstream pipeline.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"start_at": {
"anyOf": [
{
"$ref": "#/$defs/KafkaStartAt"
},
{
"type": "null"
}
],
"description": "The start position when the group has no committed position. `earliest`\nreads the topic from the start. `latest` reads only new messages. The\ndefault is `latest`."
},
"topic": {
"description": "The topic to consume from.",
"type": "string"
}
},
"required": [
"connection",
"topic",
"group"
],
"title": "kafka",
"type": "object"
},
"KafkaOutputConfig": {
"description": "Publishes each message in the batch to a kafka topic, one record for each\nmessage.\n\nThe records have no key, so kafka distributes them across the partitions\nof the topic.",
"properties": {
"connection": {
"description": "The name of the kafka connection to publish to. Declare the connection\nin the connections file.",
"type": "string",
"x-connection": "kafka"
},
"topic": {
"description": "The topic to publish to.",
"type": "string"
}
},
"required": [
"connection",
"topic"
],
"title": "kafka",
"type": "object"
},
"KafkaStartAt": {
"description": "The position at which a new consumer group starts to read.",
"enum": [
"earliest",
"latest"
],
"type": "string"
},
"KeepPolicy": {
"description": "Which fields of the input message a `map` keeps.",
"oneOf": [
{
"const": "all",
"description": "Keep all fields, and apply the mappings to them. This is the default.",
"type": "string"
},
{
"const": "mapped",
"description": "Keep only the fields that the mappings wrote. Use it to prepare a\nmessage for an output with a fixed shape, for example a `postgres`\ntable. It also removes intermediate fields. You cannot use `drop` with\n`mapped`.",
"type": "string"
}
]
},
"Literal": {
"description": "A literal value. A `constant` writes it, and a `default` writes it in place\nof a missing field. The `type` field selects the type of the value.",
"oneOf": [
{
"description": "A string.",
"properties": {
"type": {
"const": "text",
"type": "string"
},
"value": {
"description": "The text.",
"type": "string"
}
},
"required": [
"type",
"value"
],
"type": "object"
},
{
"description": "A number.",
"properties": {
"type": {
"const": "number",
"type": "string"
},
"value": {
"description": "The number.",
"format": "double",
"type": "number"
}
},
"required": [
"type",
"value"
],
"type": "object"
},
{
"description": "True or false.",
"properties": {
"type": {
"const": "boolean",
"type": "string"
},
"value": {
"description": "The boolean value.",
"type": "boolean"
}
},
"required": [
"type",
"value"
],
"type": "object"
},
{
"description": "JSON `null`. The field is present with the value `null`.",
"properties": {
"type": {
"const": "null",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
}
]
},
"MapMissingPolicy": {
"description": "What `map` does with a message that does not have a field that a mapping\nreads.\n\nNo value drops the message. To drop a message, use `filter`.",
"oneOf": [
{
"const": "error",
"description": "Fail the batch. This is the default. To accept a missing field, use\n`omit`, or give the mapping a `default`.",
"type": "string"
},
{
"const": "omit",
"description": "Do not write the target field.",
"type": "string"
},
{
"const": "null",
"description": "Write the target field as `null`.",
"type": "string"
}
]
},
"MapTransformConfig": {
"description": "Changes the shape of every message. A mapping can rename, move, cast or\nremove a field, or write a constant. The transform applies the mappings in\norder.\n\nEach entry in `mappings` reads fields from the message and writes one\nfield. A mapping can read the fields that earlier mappings wrote. Use this\nfor intermediate values. With `keep: all`, a `drop` can remove them again.\n\nReads and writes use dotted paths. For example, an `as` of `sensor.id`\nwrites the value inside a `sensor` object. If the object does not exist,\nthe transform makes it.\n\nBy default, the message passes through with the mappings applied to it.\n`keep` can change this. One message goes in and one message comes out. To\ndrop a message, use `filter`. To make many messages, use `splitter`.",
"properties": {
"keep": {
"$ref": "#/$defs/KeepPolicy",
"description": "Which fields of the input message the output keeps. The default is\n`all`."
},
"mappings": {
"description": "The fields to write, in order. Give one mapping or more. Two mappings\nmust not write the same field.",
"items": {
"$ref": "#/$defs/Mapping"
},
"type": "array"
},
"on_missing": {
"$ref": "#/$defs/MapMissingPolicy",
"description": "What to do with a message that does not have a field that a mapping\nreads. The default is `error`. A `default` on the mapping applies\nfirst. Use a `default` when you expect one field to be absent."
}
},
"required": [
"mappings"
],
"title": "map",
"type": "object"
},
"Mapping": {
"description": "One field that the transform writes onto the message, and the source of\nits value. The `type` field selects the mapping.",
"oneOf": [
{
"description": "Reads a value from one field and writes it to another field. Use it to\nrename a field, or to move a field out of a nested object, for example\nfrom `_meta.subject` to `subject`.",
"properties": {
"as": {
"description": "The field to write. The default is the last segment of `from`.",
"type": [
"string",
"null"
]
},
"default": {
"anyOf": [
{
"$ref": "#/$defs/Literal"
},
{
"type": "null"
}
],
"description": "The value to write when the message does not have `from`. With it,\n`on_missing` does not apply."
},
"from": {
"description": "The field to read, as a dotted path.",
"type": "string"
},
"type": {
"const": "copy",
"type": "string"
}
},
"required": [
"type",
"from"
],
"type": "object"
},
{
"description": "Writes a fixed value, for example the name of the site.",
"properties": {
"as": {
"description": "The field to write the value to.",
"type": "string"
},
"type": {
"const": "constant",
"type": "string"
},
"value": {
"$ref": "#/$defs/Literal",
"description": "The value to write."
}
},
"required": [
"type",
"value",
"as"
],
"type": "object"
},
{
"description": "Writes the value of the first field in a list that the message has.\nUse it when two sources use different names for the same field.",
"properties": {
"as": {
"description": "The field to write the first value to.",
"type": "string"
},
"default": {
"anyOf": [
{
"$ref": "#/$defs/Literal"
},
{
"type": "null"
}
],
"description": "The value to write when the message has none of the fields."
},
"from": {
"description": "The fields to try, in order. Give two fields or more.",
"items": {
"type": "string"
},
"type": "array"
},
"type": {
"const": "coalesce",
"type": "string"
}
},
"required": [
"type",
"from",
"as"
],
"type": "object"
},
{
"description": "Converts a value from one JSON type to another. For example, it\nconverts the string `\"12.5\"` to the number `12.5`, or epoch seconds to\na timestamp. It can also parse a string that contains JSON.\n\nThis is the only place in kayak that converts a value. The column\nmapping of the database outputs checks a value and does not convert it.",
"properties": {
"as": {
"description": "The field to write. The default is the last segment of `from`. For\nexample, `{\"from\": \"value\", \"to\": \"float\"}` converts `value` in place.",
"type": [
"string",
"null"
]
},
"default": {
"anyOf": [
{
"$ref": "#/$defs/Literal"
},
{
"type": "null"
}
],
"description": "The value to write when the message does not have `from`. A value\nthat is present and does not convert is always an error."
},
"from": {
"description": "The field to read.",
"type": "string"
},
"to": {
"$ref": "#/$defs/CastType",
"description": "The type to convert the value to."
},
"type": {
"const": "cast",
"type": "string"
}
},
"required": [
"type",
"from",
"to"
],
"type": "object"
},
{
"description": "Joins fields and literal text into one string. For example, use it to\nwrite a `site/machine` key for `group_by`.",
"properties": {
"as": {
"description": "The field to write the string to.",
"type": "string"
},
"parts": {
"description": "The parts, in order. Give one part or more.",
"items": {
"$ref": "#/$defs/ConcatPart"
},
"type": "array"
},
"type": {
"const": "concat",
"type": "string"
}
},
"required": [
"type",
"parts",
"as"
],
"type": "object"
},
{
"description": "One arithmetic operation on two numbers. Each number is a field or a\nliteral.\n\nFor a calculation with more steps, use more mappings with intermediate\nfields. For example, `(f - 32) / 1.8` is two mappings. For a long\ncalculation, use the `script` transform.",
"properties": {
"as": {
"description": "The field to write the result to.",
"type": "string"
},
"left": {
"$ref": "#/$defs/Operand",
"description": "The left operand."
},
"on_zero": {
"$ref": "#/$defs/OnZero",
"description": "For `divide`: what to do when the right field holds zero. The\ndefault is `error`, which fails the batch."
},
"operator": {
"$ref": "#/$defs/ArithmeticOperator",
"description": "The operation to do."
},
"right": {
"$ref": "#/$defs/Operand",
"description": "The right operand."
},
"type": {
"const": "arithmetic",
"type": "string"
}
},
"required": [
"type",
"left",
"operator",
"right",
"as"
],
"type": "object"
},
{
"description": "Writes the start of the calendar period that a time is in, for example\nthe hour, the day or the shift.\n\nUse the field in `group_by`. A stateful transform that groups by it\nkeeps one series for each period. The idle timeout of the state bucket\nremoves the old periods.\n\nThe periods use the local clock of `timezone`. Thus, a shift that\nstarts at 06:00 starts at 06:00 in summer and in winter. On the night\nthat the clock changes, the shift is 7 or 9 hours.\n\nThe periods start at local midnight on 1 January 1970, plus\n`offset_seconds`. For example, `every_seconds: 28800` with\n`offset_seconds: 21600` gives 06:00, 14:00 and 22:00. A week starts on a\nThursday. For a week that starts on a Monday, add an offset of four days.",
"properties": {
"as": {
"description": "The field to write the start time to.",
"type": "string"
},
"every_seconds": {
"description": "The length of a period, in seconds.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"format": {
"anyOf": [
{
"$ref": "#/$defs/TimeFormat"
},
{
"type": "null"
}
],
"description": "The format of the start time. The default is `rfc3339`."
},
"from": {
"description": "The field that holds the time, as an RFC 3339 string or as\nmilliseconds since the epoch.",
"type": "string"
},
"offset_seconds": {
"description": "The offset of the start of the periods, in seconds. It must be\nless than `every_seconds`.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"timezone": {
"description": "The IANA time zone of the periods, for example `Europe/Stockholm`.\nThe default is UTC.",
"type": [
"string",
"null"
]
},
"type": {
"const": "time_bucket",
"type": "string"
}
},
"required": [
"type",
"from",
"every_seconds",
"as"
],
"type": "object"
},
{
"description": "Removes fields from the message. Use it to remove metadata fields\nbefore the output. A field that is not there is not an error.\n`on_missing` does not apply.",
"properties": {
"from": {
"description": "The fields to remove. Give one field or more.",
"items": {
"type": "string"
},
"type": "array"
},
"type": {
"const": "drop",
"type": "string"
}
},
"required": [
"type",
"from"
],
"type": "object"
}
]
},
"MissingColumnPolicy": {
"description": "What to do with a message that does not have the field of a column. A\nfield with the value `null` counts as missing.",
"oneOf": [
{
"const": "null",
"description": "Write `NULL`. This is the default for a nullable column.",
"type": "string"
},
{
"const": "error",
"description": "Fail the batch. This is the default for a column with\n`\"nullable\": false`.",
"type": "string"
},
{
"const": "skip_row",
"description": "Do not write the message. The output writes no column of that row.",
"type": "string"
}
]
},
"MissingFieldPolicy": {
"description": "What to do with a message that does not have a field that is aggregated or\ngrouped by. A field with the value `null` is missing.",
"oneOf": [
{
"const": "error",
"description": "Fail the batch. The default. A sum of only some of the messages gives a\nwrong result that is not visible downstream.",
"type": "string"
},
{
"const": "skip",
"description": "Do not use that message in that aggregation. An aggregation with no\nvalues gives `null`, or `0` for the counts.",
"type": "string"
}
]
},
"MqttConfig": {
"description": "Subscribes to an mqtt topic and parses each message as JSON.\n\nThe topic can be a filter with the mqtt wildcards `+` and `#`. The input\nskips a payload that is not JSON and writes a warning to the log.\n\nThe input opens the connection on the first read. kayak makes the client id\nfrom the pipeline id and the topic. You cannot set the client id.",
"properties": {
"connection": {
"description": "The name of the mqtt connection to subscribe on. Declare the connection\nin the connections file.",
"type": "string",
"x-connection": "mqtt"
},
"max_batch": {
"description": "The maximum number of messages in one batch. The default is 1.\n\nThe input puts only messages that are already received into a batch.\nIt does not wait for more messages, so a high value does not add\nlatency on a quiet topic.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"qos": {
"anyOf": [
{
"$ref": "#/$defs/MqttQos"
},
{
"type": "null"
}
],
"description": "The quality of service for the subscription. The default is\n`at_most_once`. `ack: on_delivery` requires `at_least_once` or\n`exactly_once`."
},
"topic": {
"description": "The topic or topic filter to subscribe to.",
"type": "string"
}
},
"required": [
"connection",
"topic"
],
"title": "mqtt",
"type": "object"
},
"MqttOutputConfig": {
"description": "Publishes each message in the batch to an mqtt topic, one message for each\npublish.\n\nkayak makes the client id from the pipeline id and the topic. You cannot\nset the client id.",
"properties": {
"connection": {
"description": "The name of the mqtt connection to publish on. Declare the connection\nin the connections file.",
"type": "string",
"x-connection": "mqtt"
},
"qos": {
"anyOf": [
{
"$ref": "#/$defs/MqttQos"
},
{
"type": "null"
}
],
"description": "The quality of service for each publish. The default is `at_most_once`."
},
"retain": {
"description": "Tell the broker to keep the message as the retained message of the\ntopic. The broker sends it to each new subscriber. The default is false.",
"type": [
"boolean",
"null"
]
},
"topic": {
"description": "The topic to publish to.",
"type": "string"
}
},
"required": [
"connection",
"topic"
],
"title": "mqtt",
"type": "object"
},
"MqttQos": {
"description": "The mqtt quality of service for a subscribe or a publish.",
"oneOf": [
{
"const": "at_most_once",
"description": "QoS 0. The broker does not send a message again and there is no\nacknowledgement. The default.",
"type": "string"
},
{
"const": "at_least_once",
"description": "QoS 1. The broker sends a message again until it gets an\nacknowledgement, so a message can arrive more than one time. An input\nwith `ack: on_delivery` requires this level or higher.",
"type": "string"
},
{
"const": "exactly_once",
"description": "QoS 2. A four-part handshake makes sure of exactly one delivery. This\nlevel has the highest cost. Use `at_least_once` if a duplicate message\nis not a problem.",
"type": "string"
}
]
},
"NatsConfig": {
"description": "Subscribes to a nats subject and parses each message as JSON.\n\nThe input skips a payload that is not JSON and writes a warning to the log.\nThe input opens the connection on the first read.",
"properties": {
"connection": {
"description": "The name of the nats connection to subscribe on. Declare the connection\nin the connections file.",
"type": "string",
"x-connection": "nats"
},
"max_batch": {
"description": "The maximum number of messages in one batch. The default is 1.\n\nThe input puts only messages that are already received into a batch.\nIt does not wait for more messages, so a high value does not add\nlatency on a quiet subject.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"subject": {
"description": "The subject to subscribe to.",
"type": "string"
}
},
"required": [
"connection",
"subject"
],
"title": "nats",
"type": "object"
},
"NatsOutputConfig": {
"description": "Publishes each message in the batch to a nats subject, one message for each\npublish.",
"properties": {
"connection": {
"description": "The name of the nats connection to publish on. Declare the connection\nin the connections file.",
"type": "string",
"x-connection": "nats"
},
"subject": {
"description": "The subject to publish to.",
"type": "string"
}
},
"required": [
"connection",
"subject"
],
"title": "nats",
"type": "object"
},
"NumericFilterOperatorKind": {
"description": "How a `numeric` condition compares a number to the value in the config.",
"enum": [
"greater_than",
"less_than",
"equal_to",
"not_equal_to"
],
"type": "string"
},
"OnZero": {
"description": "What a `divide` does when the right field holds zero. For example, this\noccurs with a ratio over an empty period.",
"oneOf": [
{
"description": "Fail the batch. The error names the division. This is the default.",
"properties": {
"type": {
"const": "error",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"description": "Write `null` as the result.",
"properties": {
"type": {
"const": "null",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"description": "Write a number as the result.",
"properties": {
"type": {
"const": "value",
"type": "string"
},
"value": {
"description": "The number to write.",
"format": "double",
"type": "number"
}
},
"required": [
"type",
"value"
],
"type": "object"
}
]
},
"OpcuaBrowseConfig": {
"description": "All variables under a node in the address space of the server. The input\nbrowses for them when the pipeline starts.\n\nThe input reads the address space only at the start. A tag that is added\nto the server later is read only after a restart. A tag that is removed\nstops without an error. Use a `nodes` list to name in the config file\nexactly which nodes the input reads. You can use `browse` and `nodes`\ntogether.",
"properties": {
"depth": {
"description": "The number of levels below the root to follow. The default is 3. The\nvalue 0 is not permitted. There is no value for \"all levels\", because\nthe address space of a plant server can have thousands of nodes.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"root": {
"description": "The id of the node to browse under, in the same notation as `node_id`.\nThis is usually a folder, for example `ns=2;s=Machine1`. The input\nsubscribes to each variable under it. It follows folders and objects\nbut does not subscribe to them.",
"type": "string"
}
},
"required": [
"root"
],
"title": "opcua browse",
"type": "object"
},
"OpcuaConfig": {
"description": "Subscribes to variables on an OPC UA server and sends one message for each\nchange of a value.\n\nThe input makes a subscription with one monitored item for each node. The\nserver sends a value when it changes. The input does not poll. A tag that\ndoes not change sends no messages. `publish_interval_ms` sets how\nfrequently the server can send. It does not set how frequently the server\nsamples.\n\nEach message is one reading. It contains the tag and the value:\n\n```json\n{\n \"node\": \"ns=2;s=Machine1.Temperature\",\n \"name\": \"temperature\",\n \"value\": 21.5,\n \"status\": \"Good\",\n \"source_timestamp\": \"2026-01-01T12:00:00.123Z\",\n \"server_timestamp\": \"2026-01-01T12:00:00.130Z\"\n}\n```\n\n`status` is the quality of the reading and is always present. A failed\nsensor sends a `Bad...` status with a `null` value. Use a `filter` to remove\nthese readings. `source_timestamp` is the time at which the device produced\nthe value. Use it to reduce or partition. The `received_at` field of the\nenvelope is the time at which kayak read the value.\n\nSet `nodes`, `browse` or both. One of them is required. The input subscribes\nto a node only one time, also when two settings name it.",
"properties": {
"browse": {
"anyOf": [
{
"$ref": "#/$defs/OpcuaBrowseConfig"
},
{
"type": "null"
}
],
"description": "A node to browse. The input subscribes to each variable under it."
},
"connection": {
"description": "The name of the opcua connection to subscribe on. Declare the\nconnection in the connections file.",
"type": "string",
"x-connection": "opcua"
},
"deadband": {
"description": "The smallest change of a value that the server reports, in the units of\nthe value. If you do not set it, the server reports each change.\n\nThe server applies the deadband, so it decreases network traffic. It\napplies only to numeric nodes. The server reports each change of a\nstring or a boolean.",
"format": "double",
"type": [
"number",
"null"
]
},
"max_batch": {
"description": "The maximum number of messages in one batch. The default is 1.\n\nOne publish from the server contains each node that changed in the\ninterval. With 200 tags at 1 Hz and the default, the pipeline handles\n200 batches each second. A higher value decreases this cost. The input\nputs only changes that are already received into a batch.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"nodes": {
"description": "The nodes to subscribe to, one entry for each node.",
"items": {
"$ref": "#/$defs/OpcuaNodeConfig"
},
"type": "array"
},
"publish_interval_ms": {
"description": "The interval at which the server can send a group of changes, in ms.\nThe default is 1000. This value sets the longest time that a change\nwaits.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"queue_size": {
"description": "The number of samples that the server keeps for one node between two\npublishes. The default is 1. With 1, the server sends only the latest\nvalue of a node that changes two times in one interval. Increase it,\ntogether with `sampling_interval_ms`, when you need each sample.",
"format": "uint32",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"sampling_interval_ms": {
"description": "The interval at which the server samples each node, in ms. If you do not\nset it, the server samples at the publish interval. Set a smaller value\nto get more readings between two publishes.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
}
},
"required": [
"connection"
],
"title": "opcua",
"type": "object"
},
"OpcuaNodeConfig": {
"description": "One node that an `opcua` input subscribes to, and the name for it in the\nmessages.",
"properties": {
"name": {
"description": "The name of the node in the messages. The default is the node id. Set\na readable name to use in a `group_by` or a column mapping.",
"type": [
"string",
"null"
]
},
"node_id": {
"description": "The id of the node, in OPC UA notation. Use `ns=2;s=Machine1.Temperature`\nfor a string identifier and `ns=2;i=1042` for a numeric identifier. Use\n`g=` for a GUID and `b=` for an opaque identifier. A node id with no\n`ns=` is in namespace 0, the namespace of the server.",
"type": "string"
}
},
"required": [
"node_id"
],
"title": "opcua node",
"type": "object"
},
"Operand": {
"description": "One side of an [`Mapping::Arithmetic`]: a field to read, or a fixed number.",
"oneOf": [
{
"description": "A number from the message.",
"properties": {
"field": {
"description": "The field to read. It must hold a number.",
"type": "string"
},
"type": {
"const": "field",
"type": "string"
}
},
"required": [
"type",
"field"
],
"type": "object"
},
{
"description": "A number in the config.",
"properties": {
"type": {
"const": "value",
"type": "string"
},
"value": {
"description": "The number.",
"format": "double",
"type": "number"
}
},
"required": [
"type",
"value"
],
"type": "object"
}
]
},
"OutputConfig": {
"oneOf": [
{
"$ref": "#/$defs/StdoutOutputConfig",
"properties": {
"type": {
"const": "stdout",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/FileOutputConfig",
"properties": {
"type": {
"const": "file",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/S3OutputConfig",
"properties": {
"type": {
"const": "s3",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/KafkaOutputConfig",
"properties": {
"type": {
"const": "kafka",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/NatsOutputConfig",
"properties": {
"type": {
"const": "nats",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/PostgresOutputConfig",
"properties": {
"type": {
"const": "postgres",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/ClickhouseOutputConfig",
"properties": {
"type": {
"const": "clickhouse",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/MqttOutputConfig",
"properties": {
"type": {
"const": "mqtt",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/RedisOutputConfig",
"properties": {
"type": {
"const": "redis",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/HttpOutputConfig",
"properties": {
"type": {
"const": "http",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/InduOutputConfig",
"properties": {
"type": {
"const": "indu",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/TidepoolOutputConfig",
"properties": {
"type": {
"const": "tidepool",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
}
],
"type": "object"
},
"PipelineConfig": {
"description": "Reads the output of another pipeline. Use it to connect pipelines into a\ngraph.\n\nMany pipelines can read from the same upstream. Each of them gets all\nbatches. The upstream must exist when kayak creates this pipeline, so\ndeclare the upstream first in the config file.",
"properties": {
"upstream": {
"description": "The id of the pipeline to read from.",
"type": "string",
"x-pipeline-id": true
}
},
"required": [
"upstream"
],
"title": "pipeline",
"type": "object"
},
"PipelineState": {
"description": "The state bucket of a pipeline, and the field that gives the key of each\nmessage.\n\nEach pipeline sets its own key, because the same value can have different\nfield names in two streams. For example, one stream has\n`_meta.machine_id` and another stream has `machine_id`.\n\nMake sure that all pipelines that share a bucket use keys with the same\nvalues. kayak does not check this.",
"properties": {
"bucket": {
"description": "The name of the bucket that this pipeline reads and writes. Declare the\nbucket under `state` at the top of the config. If the bucket is not\ndeclared, the pipeline does not build.",
"type": "string"
},
"key": {
"description": "The field that gives the key, as a dotted path. For example,\n`_meta.machine_id`.\n\nLeave it out for one value for the full bucket. Use that only for an\nitem that has one value. For a value for each device, set `key`.",
"type": [
"string",
"null"
]
}
},
"required": [
"bucket"
],
"title": "pipeline state",
"type": "object"
},
"PivotTransformConfig": {
"description": "Changes a stream with one reading per message into rows. The transform\nkeeps the latest value of each name in `names` for each key. It writes all\nof these values onto every message.\n\nFor example, the input is `{\"sensor\": \"state\", \"value\": \"RUNNING\"}` and\nthen `{\"sensor\": \"fault\", \"value\": \"NONE\"}`. The output row is\n`{\"state\": \"RUNNING\", \"fault\": \"NONE\", ...}`.\n\nWhen the `name` field of a message holds one of `names`, the transform\nfirst records that reading. Then it writes every value that it keeps for\nthe key onto the message. It writes them at the top level, or under\n`into`. A name with no reading yet for the key is not written. One message\ngoes in and one message comes out.\n\n`names` is required because it limits the state. A message with a name that\nis not in `names` adds nothing, but it gets the row.",
"properties": {
"group_by": {
"description": "The fields that identify a row, as in `reduce`. Leave it out for one\nrow.",
"items": {
"type": "string"
},
"type": "array"
},
"into": {
"description": "An object field to write the values under. Leave it out to write them\nat the top level.",
"type": [
"string",
"null"
]
},
"name": {
"description": "The field that holds the name of the reading. kayak compares its value\nwith `names`.",
"type": "string",
"x-message-field": true
},
"names": {
"description": "The names to keep and write. Each name becomes a field. Give one name\nor more.",
"items": {
"type": "string"
},
"type": "array"
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have a group field, or that\nhas a name from `names` and no `value`. The default is `error`."
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"value": {
"description": "The field that holds the reading. The value can be any JSON value, for\nexample a string or a number.",
"type": "string",
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"name",
"value",
"names"
],
"title": "pivot",
"type": "object"
},
"PollMode": {
"description": "Whether a read returns all rows or only the new rows.",
"oneOf": [
{
"description": "Each read returns all rows, in one query. Use it for reference data,\nfor example a table of recipes for a `remember` transform. A snapshot\nhas no page limit. Use it only for a table that fits in memory.",
"properties": {
"type": {
"const": "snapshot",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"description": "Each read returns only the rows with a `field` value above the highest\nvalue that the input already sent. The input reads pages in the order\nof the field. The value of the field must increase, for example an id\nor an `updated_at`. Put an index on the field. Without an index, each\nread scans the full table.",
"properties": {
"field": {
"description": "The column that the input follows. The watermark is the highest\nvalue that the input sent. Each read asks for the rows above the\nwatermark. The input does not read rows where the column is `null`.",
"type": "string"
},
"lag_secs": {
"description": "For a timestamp column: the time to stay behind the current time,\nin seconds. A later read gets the rows that are less than this time\nbefore `now()`. This gives late transactions time to commit. Do not\nuse it with a numeric column. The server refuses the query.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"start_from": {
"anyOf": [
{
"$ref": "#/$defs/StartFrom"
},
{
"type": "null"
}
],
"description": "Where the first read starts. With `newest`, the input reads only\nthe rows added after the pipeline started. With `oldest`, it reads\nall rows first and then follows the table. The default is `newest`."
},
"type": {
"const": "incremental",
"type": "string"
}
},
"required": [
"type",
"field"
],
"type": "object"
}
]
},
"PostgresInputConfig": {
"description": "Reads a postgres table, view or query at an interval and sends each row as\na message.\n\nThe input runs a query each `interval_secs`. In `snapshot` mode, it reads\nthe full relation. In `incremental` mode, it reads only the rows after the\nlast read, by a column that increases. Each row becomes one JSON object\nwith the column names as fields. Postgres makes the object with\n`row_to_json`. A timestamp is ISO 8601, a `numeric` keeps its digits, and\na `jsonb` column is a nested value:\n\n```json\n{\"id\": 42, \"sensor\": \"press-3\", \"value\": 21.5, \"recorded_at\": \"2026-01-01T12:00:00.123456+00:00\"}\n```\n\nAn incremental read is at-least-once across a restart. The input keeps the\nwatermark in memory, and after a restart it starts again from `start_from`.\nAn incremental read does not see a deleted row. Put an index on the column\nthat it follows, or each read scans the full table.",
"properties": {
"columns": {
"description": "The columns to read, in order. Leave it empty to read all columns. For\nan incremental input, the list must include `field`.",
"items": {
"type": "string"
},
"type": "array"
},
"connection": {
"description": "The name of the postgres connection to read through. Declare the\nconnection in the connections file.",
"type": "string",
"x-connection": "postgres"
},
"interval_secs": {
"description": "The time between two reads, in seconds. The time starts at the end of a\nread. Thus, two reads never overlap. The first read occurs when the\npipeline starts.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"max_batch": {
"description": "The maximum number of rows in one batch. The default is 1. The input\nputs rows that it already read into a batch. It does not wait for more\nrows to fill a batch.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"mode": {
"$ref": "#/$defs/PollMode",
"description": "Whether each read returns all rows (`snapshot`) or only the rows after\nthe last read (`incremental`)."
},
"page_size": {
"description": "The maximum number of rows that one query returns. The default is\n1000. When a page is full, the input reads the next page immediately.\nThe interval starts after a page that is not full. A `snapshot` ignores\nthis field and reads all rows in one query.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"query": {
"description": "A `SELECT` to read in place of a table. The input puts the query in a\nsubquery. It adds the cursor condition, the `ORDER BY` and the page\nlimit outside the subquery. Thus, the query needs no placeholder and no\n`ORDER BY`.\n\nWrite one statement with no semicolon at the end. You can use all SQL\nthat the server accepts in a subquery, for example `WITH`.",
"type": [
"string",
"null"
]
},
"table": {
"description": "The table or view to read, as `name` or `schema.name`. Give exactly one\nof `table` and `query`.",
"type": [
"string",
"null"
]
}
},
"required": [
"connection",
"interval_secs",
"mode"
],
"title": "postgres",
"type": "object"
},
"PostgresOutputConfig": {
"description": "Inserts each message in the batch into a postgres table, one row for each\nmessage.\n\nWith `columns`, each entry names a column, its type and the field to read,\nfor example `{\"name\": \"temperature\", \"type\": \"float\", \"field\": \"reading.temp_c\"}`.\nThe default `field` is the name of the column. Without `columns`, the table\nhas an `id`, a `received_at` timestamp and a `payload` column of type\n`jsonb` that contains the full message.\n\nThe output creates the table if it does not exist. Set `create_table` to\nfalse for a table that another system owns. The output does not change an\nexisting table. If the table does not agree with the columns, the insert\nfails with the error from postgres.",
"properties": {
"columns": {
"description": "The column for each message field. If you do not set it, the output\nkeeps each full message as JSON in a `payload` column.",
"items": {
"$ref": "#/$defs/ColumnMapping"
},
"type": "array"
},
"connection": {
"description": "The name of the postgres connection to insert through. Declare the\nconnection in the connections file. The connection gives the host, the\ndatabase and the role.",
"type": "string",
"x-connection": "postgres"
},
"create_table": {
"description": "Create the table on connect if it does not exist. The default is true.",
"type": [
"boolean",
"null"
]
},
"indexes": {
"description": "The indexes to create with the table. Each index names mapped columns,\nin sequence.",
"items": {
"$ref": "#/$defs/TableIndex"
},
"type": "array"
},
"on_extra_fields": {
"$ref": "#/$defs/ExtraFieldPolicy",
"description": "What to do with a message that has fields that no column reads."
},
"primary_key": {
"description": "The columns of the primary key of the created table. If you do not set\nit, the table gets an `id` and a `received_at` timestamp. If you set it,\nthe table does not get these two columns.",
"items": {
"type": "string"
},
"type": "array"
},
"table": {
"description": "The table to insert into. The output creates it if it does not exist.\nYou can add a schema (`analytics.readings`). Use only letters, digits\nand underscores.",
"type": "string"
}
},
"required": [
"connection",
"table"
],
"title": "postgres",
"type": "object"
},
"RecallMissingPolicy": {
"description": "What `recall` does when the bucket has no value for the key of a message.\n\nThe default is `skip`, because the bucket is empty when the pipeline\nstarts. With `error`, the batches fail until the bucket has values.",
"oneOf": [
{
"const": "skip",
"description": "Drop the message. The default. Without the recalled values, a reducer\nafter this transform puts all these messages into one wrong group.",
"type": "string"
},
{
"const": "null",
"description": "Send the message on with the missing names as `null`.",
"type": "string"
},
{
"const": "error",
"description": "Fail the batch. Use it only when another component always fills the\nbucket first.",
"type": "string"
}
]
},
"RecallTransformConfig": {
"description": "Writes values from the state bucket of the pipeline onto each message,\nwith the names from `remember`.\n\nUse it to add a slow fact to a fast stream, for example the current recipe\nof a machine. The values become top-level fields, so a `reducer` after it\ncan group by them.\n\nThe pipeline must have a `state`, or the transform fails to build.",
"properties": {
"on_missing": {
"$ref": "#/$defs/RecallMissingPolicy",
"description": "What to do with a message when the bucket has no value for its key.\nThe default is `skip`."
},
"recall": {
"description": "The names to read from the bucket, as `remember` wrote them. The\ntransform writes each value onto the message with the same name.",
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"recall"
],
"title": "recall",
"type": "object"
},
"RedisConfig": {
"description": "Subscribes to a redis channel and parses each message as JSON.\n\nThe input skips a payload that is not JSON and writes a warning to the log.\nThe input opens the connection on the first read.\n\nThe input uses `SUBSCRIBE`, so the channel name must be exact. Patterns are\nnot supported. Redis pub/sub does not send a message again. When the input\nis not connected, it does not receive the messages that are published.",
"properties": {
"channel": {
"description": "The channel to subscribe to.",
"type": "string"
},
"connection": {
"description": "The name of the redis connection to subscribe on. Declare the\nconnection in the connections file.",
"type": "string",
"x-connection": "redis"
},
"max_batch": {
"description": "The maximum number of messages in one batch. The default is 1.\n\nThe input puts only messages that are already received into a batch.\nIt does not wait for more messages, so a high value does not add\nlatency on a quiet channel.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
}
},
"required": [
"connection",
"channel"
],
"title": "redis",
"type": "object"
},
"RedisOutputConfig": {
"description": "Publishes each message in the batch to a redis channel, one message for\neach publish.",
"properties": {
"channel": {
"description": "The channel to publish to.",
"type": "string"
},
"connection": {
"description": "The name of the redis connection to publish on. Declare the connection\nin the connections file.",
"type": "string",
"x-connection": "redis"
}
},
"required": [
"connection",
"channel"
],
"title": "redis",
"type": "object"
},
"ReduceFnKind": {
"description": "How a reducer combines the values of one field into one result.",
"oneOf": [
{
"const": "sum",
"description": "The total. Numbers only.",
"type": "string"
},
{
"const": "avg",
"description": "The arithmetic mean. Numbers only.",
"type": "string"
},
{
"const": "min",
"description": "The smallest value. Numbers compare as numbers. Strings compare in\nalphabetical sequence, so `min` of ISO timestamps is the earliest time.",
"type": "string"
},
{
"const": "max",
"description": "The largest value. Values compare as for `min`.",
"type": "string"
},
{
"const": "count",
"description": "The number of messages. This function does not need a `field`. With a\n`field`, it counts the messages that have the field.",
"type": "string"
},
{
"const": "count_distinct",
"description": "The number of different values. The function compares the values as\nJSON.",
"type": "string"
},
{
"const": "first",
"description": "The value from the first message of the group, of any type.",
"type": "string"
},
{
"const": "last",
"description": "The value from the last message of the group.",
"type": "string"
},
{
"const": "collect",
"description": "All values as an array, in the sequence of arrival.",
"type": "string"
},
{
"const": "median",
"description": "The middle value, or the mean of the two middle values. Numbers only.",
"type": "string"
},
{
"const": "stddev",
"description": "The population standard deviation. Numbers only.",
"type": "string"
},
{
"const": "slope",
"description": "The rate of change of the field per second, from a least-squares line\nagainst the time of each message. Numbers only. The reducer must have a\n`time` setting, or it fails to build.",
"type": "string"
}
]
},
"ReduceTransformConfig": {
"description": "Reduces a batch to one message for each group. Put a buffer before it, or\nit gets only one message at a time.\n\nWithout `group_by`, the full batch is one group and the reducer sends one\nmessage. With `group_by`, it sends one message for each different\ncombination of those fields. The messages are in the sequence in which the\ngroups first occur. Each message contains the `group_by` fields and the\nresults of the aggregations.\n\nEach aggregation has a `function`, the `field` to use and the name `as` for\nthe result, for example\n`{\"function\": \"avg\", \"field\": \"value\", \"as\": \"mean\"}`.",
"properties": {
"aggregations": {
"description": "The values to calculate. At least one aggregation is required. Each one\nmust have a different `as`.",
"items": {
"$ref": "#/$defs/Aggregation"
},
"type": "array"
},
"group_by": {
"description": "The fields whose combination defines a group. If you do not set it, the\nfull batch is one group.",
"items": {
"type": "string"
},
"type": "array"
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have one of the fields. The\ndefault is `error`."
},
"time": {
"description": "The field that contains the time of each message, as an RFC 3339 string\nor as ms since the epoch. `slope` requires it. A message without this\nfield fails the batch. If you do not set it, the time of each message is\nits arrival time.",
"type": [
"string",
"null"
],
"x-message-field": true
}
},
"required": [
"aggregations"
],
"title": "reducer",
"type": "object"
},
"RememberTransformConfig": {
"description": "Writes values from the messages that match into the state bucket of the\npipeline. The key is the field that `state.key` of the pipeline names.\n\nThe transform sends each message on without changes. It does not filter.\n\nThe pipeline must have a `state`, or the transform fails to build.",
"properties": {
"remember": {
"description": "The values to take from a message that matches. At least one entry is\nrequired. Each entry must have a different `as`.",
"items": {
"$ref": "#/$defs/Remembered"
},
"type": "array"
},
"when": {
"description": "The conditions that a message must match to be remembered. All of them\nmust match. If you do not set it, the transform remembers values from\neach message. Set it when the stream contains more than one type of\nmessage.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"remember"
],
"title": "remember",
"type": "object"
},
"Remembered": {
"description": "One value to write into the state bucket of the pipeline, and its name\nthere.",
"properties": {
"as": {
"description": "The name for the value in the bucket. `recall` reads the value by this\nname. Each entry must have a different name.",
"type": "string"
},
"field": {
"description": "The field to take the value from.",
"type": "string"
}
},
"required": [
"field",
"as"
],
"type": "object"
},
"ResampleMethod": {
"description": "How the transform calculates the value at a grid point from the readings in\nits interval, and what it does with an empty interval.",
"oneOf": [
{
"const": "last",
"description": "The last reading in the interval. An empty interval sends nothing.",
"type": "string"
},
{
"const": "mean",
"description": "The mean of the readings in the interval. An empty interval sends\nnothing.",
"type": "string"
},
{
"const": "linear",
"description": "The value on a straight line between the last reading before the grid\npoint and the first reading after it. The transform sends the grid\npoint when the reading after it arrives. The same line fills the empty\nintervals between the two readings.",
"type": "string"
},
{
"const": "forward_fill",
"description": "The last reading, carried forward. An empty interval repeats the last\nvalue for a maximum of `max_gap_seconds`, and then stops. This is the\nonly method that sends values from a quiet series. Use it to change a\nsignal that reports on change into a regular signal.",
"type": "string"
}
]
},
"ResampleTransformConfig": {
"description": "Puts a series onto a regular grid. The transform sends one message for each\nkey in each `interval_seconds`, at times that are multiples of the interval.\nThe rate of the readings has no effect on the grid.\n\nThe message that the transform sends contains:\n\n- the group fields, under their leaf names,\n- the grid time as an RFC 3339 string, under the name of the `time` field\n (or `time` when the transform uses the arrival time),\n- the value, under `as` (the leaf of `field` when you leave it out).\n\nThe transform sends a grid point when a later reading arrives. With\n`forward_fill` and the arrival time, it also sends a grid point when the\nclock passes it.",
"properties": {
"as": {
"description": "The field to write the value to. The default is the leaf of `field`.",
"type": [
"string",
"null"
]
},
"field": {
"description": "The numeric field to resample.",
"type": "string",
"x-message-field": true
},
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out for\none series.",
"items": {
"type": "string"
},
"type": "array"
},
"interval_seconds": {
"description": "The distance between two grid points, in seconds.",
"format": "double",
"type": "number"
},
"max_gap_seconds": {
"description": "For `forward_fill`: the maximum time to repeat a value into empty\nintervals, in seconds. Leave it out to repeat the value with no limit.",
"format": "double",
"type": [
"number",
"null"
]
},
"method": {
"$ref": "#/$defs/ResampleMethod",
"description": "How the transform calculates the value of an interval."
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have the field or a group\nfield. The default is `error`."
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"time": {
"description": "The field that holds the time of each message, as an RFC 3339 string\nor as milliseconds since the epoch. Leave it out to use the arrival\ntime. With the arrival time, the clock finds the empty intervals.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"field",
"interval_seconds",
"method"
],
"title": "resample",
"type": "object"
},
"RollingTransformConfig": {
"description": "Writes aggregations over the last messages of the series onto each\nmessage. The aggregations use the same `{function, field, as}` list and\nfunctions as `reduce`. One message goes in and one message comes out.\n\nThe window holds a maximum of `size` messages. `size` is always required.\nWith `seconds`, the window also drops messages that are older than that\ntime, from the `time` field.\n\nHere, `count` needs a `field`. It counts the messages in the window that\nhave the field. When the window is full, the count is `size`.",
"properties": {
"aggregations": {
"description": "The aggregations to calculate over the window. Give one or more, each\nwith a different `as`.",
"items": {
"$ref": "#/$defs/Aggregation"
},
"type": "array"
},
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out for\none series.",
"items": {
"type": "string"
},
"type": "array"
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have an aggregated field or a\ngroup field. The default is `error`."
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"seconds": {
"description": "Also drop the messages that are older than this number of seconds,\nfrom the `time` field.",
"format": "double",
"type": [
"number",
"null"
]
},
"size": {
"description": "The maximum number of messages in the window.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"time": {
"description": "The field that holds the time of each message, as an RFC 3339 string\nor as milliseconds since the epoch. Leave it out to use the arrival\ntime.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"aggregations",
"size"
],
"title": "rolling",
"type": "object"
},
"RotationConfig": {
"description": "When the output closes a file and starts the next file.\n\nThe two triggers are optional. The first trigger that is reached rotates\nthe file. With no trigger, the output writes one file while the pipeline\nruns. The `file` and `s3` outputs use the same rotation settings.",
"properties": {
"interval_secs": {
"description": "Close the file this number of seconds after the output opened it. The\ntime starts when the file opens, not at the last write.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"max_rows": {
"description": "Close the file when it contains this number of messages. The output\ndoes not divide a batch, so a file can contain more messages.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
}
},
"type": "object"
},
"S3OutputConfig": {
"description": "Writes each batch to objects under a prefix in an S3-compatible bucket.\n\nThe `s3` output uses the same file names, formats and rotation as the\n`file` output. An object store cannot append to an object. Thus the output\nkeeps the current object in memory and uploads it when it rotates.\n`rotate` is required. It sets how frequently objects appear and how much\nmemory the pipeline uses.",
"properties": {
"connection": {
"description": "The name of the s3 connection to write through. Declare the connection\nin the connections file. The connection gives the bucket and the\ncredentials.",
"type": "string",
"x-connection": "s3"
},
"format": {
"anyOf": [
{
"$ref": "#/$defs/FileFormat"
},
{
"type": "null"
}
],
"description": "The layout of the messages. The default is `ndjson`."
},
"prefix": {
"description": "The key prefix to write under, for example `orders`. The output writes\neach object to `<prefix>/<part name>`. Set an empty prefix to write at\nthe root of the bucket.",
"type": "string"
},
"rotate": {
"$ref": "#/$defs/RotationConfig",
"description": "When to close an object and start the next object. Required. A\n`rotate` with no trigger fails to build."
}
},
"required": [
"connection",
"prefix",
"rotate"
],
"title": "s3",
"type": "object"
},
"ScriptScope": {
"description": "Whether the script gets one message or the full batch.\n\n`message` is the default. The operation limit then applies to each message,\nand the batch keeps its structure.\n\nUse `batch` for work on the full batch. For example, remove duplicates in\nthe batch, or calculate a value that `reduce` has no function for.",
"oneOf": [
{
"const": "message",
"description": "The script runs one time for each message, with the message in `msg`.",
"type": "string"
},
{
"const": "batch",
"description": "The script runs one time for each batch, with the messages in `batch`\nas an array. When you emit an array, the transform sends a batch of\nthose messages.",
"type": "string"
}
]
},
"ScriptSource": {
"description": "The source of the script text: `inline` or `file`.\n\nUse `inline` to send a script through the HTTP API. In a YAML config, an\ninline script is a literal block. Use `file` to keep the script in its own\nfile, where an editor, a formatter and a test can use it. A save keeps the\n`file` reference in the config. It does not copy the script into the\nconfig.",
"oneOf": [
{
"description": "The script text, in the config. Use a YAML config for an inline script.\nYAML keeps the script as a literal block. JSON must escape each newline.",
"properties": {
"code": {
"description": "The rhai source.",
"type": "string",
"x-script": "rhai"
},
"type": {
"const": "inline",
"type": "string"
}
},
"required": [
"type",
"code"
],
"type": "object"
},
{
"description": "A path to a `.rhai` file, relative to the directory of the config file.\n\nkayak reads the file and its imports when it builds the pipeline. After\nyou change the file, do a revert to use the change. A server with no\nconfig file does not accept a `file` source. Inline scripts work on that\nserver, but their imports do not.",
"properties": {
"path": {
"description": "The path, relative to the directory of the config file. The path\nmust stay in that directory.",
"type": "string"
},
"type": {
"const": "file",
"type": "string"
}
},
"required": [
"type",
"path"
],
"type": "object"
}
]
},
"ScriptTransformConfig": {
"description": "Runs a [rhai](https://rhai.rs) script over each message, or over the full\nbatch, and sends the values that the script emits.\n\nThe script gets the message as `msg`, and sends values with `emit(value)`.\nCall `emit` zero times to drop the message, one time to replace it, or more\ntimes to split it. Use a script when `filter`, `map` and `splitter` are not\nsufficient.\n\nkayak **compiles the script when it builds the pipeline**. Thus, a syntax\nerror stops the pipeline from starting. An error that occurs only with a\nmessage fails that batch. For example, a missing field or a value that does\nnot convert fails the batch.\n\nEach run of the script has an **operation limit** (`max_operations`). When\nthe script reaches the limit, kayak stops it and fails the batch. Thus, a\nscript with an endless loop cannot block the pipeline.\n\nA script can **`import`** other rhai files. Give a literal path relative to\nthe directory of the config file. The path must stay in that directory. You\ncan leave out the `.rhai` extension. kayak reads the imports when it builds\nthe pipeline. Thus, a bad import stops the pipeline from starting, and a\nrunning script does not read the filesystem.",
"properties": {
"max_operations": {
"description": "The maximum number of rhai operations in one run of the script. At the\nlimit, kayak stops the script and fails the batch. The default is\n100000. Increase it for a script that walks a large array.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"scope": {
"$ref": "#/$defs/ScriptScope",
"description": "Whether the script gets one message at a time or the full batch. The\ndefault is `message`."
},
"source": {
"$ref": "#/$defs/ScriptSource",
"description": "The script: inline in the config, or in a file beside the config."
}
},
"required": [
"source"
],
"title": "script",
"type": "object"
},
"Secret": {
"description": "A text value that can refer to secrets with `${NAME}` references.\n\nThe value is an ordinary JSON string. When kayak builds the pipeline, it\nreplaces each `${NAME}` reference with the value from the secret store of\nthe server:\n\n```json\n{ \"type\": \"nats\", \"urls\": \"nats://app:${NATS_PASSWORD}@broker:4222\" }\n```\n\nThe config keeps only the reference, never the value. Thus you can commit\nthe file, and `GET /api/pipelines` does not show the secret.\n\nkayak uses a value with no `${...}` reference as it is.",
"type": "string"
},
"SmoothMethod": {
"description": "How the transform smooths a value against the values before it.",
"oneOf": [
{
"description": "An exponentially weighted moving average. It needs no window. Give\nexactly one of `alpha`, `half_life` or `tau_seconds`.\n\n`alpha` and `half_life` count messages. Use them when messages arrive\nat a steady rate. `tau_seconds` counts time. The weight of a value is\n`1 − e^(−Δt/τ)`, where Δt is the time since the previous value. Thus, a\nreading after a long gap has more weight. Use `tau_seconds` for a sensor\nthat reports on change, or for a stream that stops for periods.",
"properties": {
"alpha": {
"description": "The weight of the newest value, from 0 to 1.",
"format": "double",
"type": [
"number",
"null"
]
},
"half_life": {
"description": "The number of messages after which the weight of a value is half.",
"format": "double",
"type": [
"number",
"null"
]
},
"tau_seconds": {
"description": "The time constant in seconds. After this time, the weight of an old\nvalue is about 37%. It reads the `time` field of the transform.",
"format": "double",
"type": [
"number",
"null"
]
},
"type": {
"const": "ewma",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"description": "The median of the last `size` values, with this value included. It\nremoves a spike of one sample.",
"properties": {
"size": {
"description": "The number of values in the window.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"type": {
"const": "median",
"type": "string"
}
},
"required": [
"type",
"size"
],
"type": "object"
},
{
"description": "A Hampel filter. When the value is more than `threshold` robust\nstandard deviations from the median of the window, the median replaces\nit. Otherwise the value stays. Use it before `detect` to remove\noutliers.",
"properties": {
"size": {
"description": "The number of values in the window, with this value included.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"threshold": {
"description": "The number of scaled MADs from the median that makes an outlier.\nThe default is `3`.",
"format": "double",
"type": [
"number",
"null"
]
},
"type": {
"const": "hampel",
"type": "string"
}
},
"required": [
"type",
"size"
],
"type": "object"
},
{
"description": "A Savitzky–Golay filter. It fits a polynomial of degree `order` to the\nlast `size` values by least squares, and gives its value at the newest\npoint. It keeps the shape of peaks. The window uses only earlier\nvalues. Until the window holds more than `order` values, the value\npasses unchanged.",
"properties": {
"order": {
"description": "The degree of the polynomial. It must be less than `size`. The\ndefault is `2`.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"size": {
"description": "The number of values in the window, with this value included.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"type": {
"const": "savitzky_golay",
"type": "string"
}
},
"required": [
"type",
"size"
],
"type": "object"
}
]
},
"SmoothTransformConfig": {
"description": "Smooths a numeric field against the earlier values of its series. The\ntransform writes the result into the field, or under `as`.\n\nAll methods except `ewma` with `tau_seconds` use the order of the values,\nand not their time. Thus, `time` is permitted only with `ewma` and\n`tau_seconds`. With other methods, `time` is an error when the pipeline\nbuilds.",
"properties": {
"as": {
"description": "The field to write the smoothed value to. Leave it out to replace the\nvalue of `field`.",
"type": [
"string",
"null"
]
},
"field": {
"description": "The numeric field to smooth.",
"type": "string",
"x-message-field": true
},
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out for\none series.",
"items": {
"type": "string"
},
"type": "array"
},
"method": {
"$ref": "#/$defs/SmoothMethod",
"description": "The method to smooth the value with."
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have the field or a group\nfield. The default is `error`."
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"time": {
"description": "For `ewma` with `tau_seconds` only: the field that holds the time of\neach message, as an RFC 3339 string or as milliseconds since the epoch.\nLeave it out to use the arrival time.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"field",
"method"
],
"title": "smooth",
"type": "object"
},
"SplitterTransformConfig": {
"description": "Divides one batch into smaller batches.\n\nThe last batch contains the messages that remain and can be smaller. For\nexample, 4 messages with `out_size: 3` give a batch of 3 and a batch of 1.",
"properties": {
"out_size": {
"description": "The number of messages in each sent batch.",
"format": "uint",
"minimum": 0,
"type": "integer"
}
},
"required": [
"out_size"
],
"title": "splitter",
"type": "object"
},
"StartFrom": {
"description": "Where the first read of an incremental input starts.",
"oneOf": [
{
"const": "oldest",
"description": "From the start. The first read returns all rows, one page at a time.",
"type": "string"
},
{
"const": "newest",
"description": "From now. The first read finds the highest value of the field and\nreturns only the rows above it. This is the default.",
"type": "string"
}
]
},
"StdoutOutputConfig": {
"description": "Prints each batch to the standard output of the server. Use it to test a\npipeline. It has no settings.",
"title": "stdout",
"type": "object"
},
"StringFilterOperatorKind": {
"description": "How a `string` condition compares a string to the value in the config.",
"enum": [
"equal_to",
"not_equal_to",
"contains"
],
"type": "string"
},
"TableIndex": {
"description": "An index to make with the table. The output makes the index only when it\nmakes the table, with `IF NOT EXISTS`.",
"properties": {
"columns": {
"description": "The columns to index, in order. Each column must be a mapped column.",
"items": {
"type": "string"
},
"type": "array"
},
"unique": {
"description": "Whether the index is unique. The default is false.",
"type": [
"boolean",
"null"
]
}
},
"required": [
"columns"
],
"title": "index",
"type": "object"
},
"ThrottleTransformConfig": {
"description": "Passes a maximum of one message per key in each period of `seconds`, and\ndrops the other messages.\n\nThe first message for each key passes. The next message that passes is the\nfirst one that arrives `seconds` or more after it. The transform drops all\nmessages between them. The period starts at the message that passed. It is\nnot aligned to the clock.\n\nThe transform does not keep messages to send later. A key that becomes\nquiet sends nothing until its next message. Use `resample` when you need the\nlast value of each period, or a value from a quiet key.\n\n`throttle` does not read a value. It passes complete messages with all\ntheir fields.",
"properties": {
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out for\none series.",
"items": {
"type": "string"
},
"type": "array"
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have a group field. The\ndefault is `error`."
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"seconds": {
"description": "The minimum time between two messages that pass for one key, in\nseconds.",
"format": "double",
"type": "number"
},
"time": {
"description": "The field that holds the time of each message, as an RFC 3339 string\nor as milliseconds since the epoch. Leave it out to use the arrival\ntime.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"seconds"
],
"title": "throttle",
"type": "object"
},
"TidepoolOutputConfig": {
"description": "Writes each batch into a Tidepool table, one request for each batch.\n\nThe table must exist in the Tidepool project. On start, the output checks\nthe columns against the table. Each mapped column must be in the table and\nhave a type that the mapping can write. Each required column of the table\nmust be written. If the check fails, the start fails.\n\n`columns` has the same format as on the database outputs. If you do not set\nit, the output sends each message as a row without changes. Tidepool checks\neach value and refuses a batch that has a problem.\n\nA refused batch fails. The error gives the problems by row and column. The\noutput tries again when the server is busy (`503`), when the server cannot\nbe reached and on other `5xx` errors. It tries again for up to\n`retry_seconds`. Each try uses the same idempotency key, so Tidepool does\nnot write a batch two times.",
"properties": {
"columns": {
"description": "The column for each message field. If you do not set it, the output\nsends each message as a row without changes.",
"items": {
"$ref": "#/$defs/ColumnMapping"
},
"type": "array"
},
"connection": {
"description": "The name of the tidepool connection to write through. Declare the\nconnection in the connections file.",
"type": "string",
"x-connection": "tidepool"
},
"on_extra_fields": {
"$ref": "#/$defs/ExtraFieldPolicy",
"description": "What to do with a message that has fields that no column reads."
},
"retry_seconds": {
"description": "The maximum time to try one batch again while the server is busy or\ncannot be reached, in s. The default is 30.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"table": {
"description": "The table to write into, with the name from the Tidepool project.",
"type": "string"
},
"timeout_seconds": {
"description": "The maximum time for one request, in s. The default is 30.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
}
},
"required": [
"connection",
"table"
],
"title": "tidepool",
"type": "object"
},
"TimeFormat": {
"description": "How a [`Mapping::TimeBucket`] writes a time.",
"oneOf": [
{
"const": "rfc3339",
"description": "An RFC 3339 string in UTC, with milliseconds. kayak writes all times in\nthis format.",
"type": "string"
},
{
"const": "millis",
"description": "Milliseconds since the epoch, as a number.",
"type": "string"
}
]
},
"TransformConfig": {
"oneOf": [
{
"$ref": "#/$defs/BufferTransformConfig",
"properties": {
"type": {
"const": "buffer",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/HttpTransformConfig",
"properties": {
"type": {
"const": "http",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/SplitterTransformConfig",
"properties": {
"type": {
"const": "splitter",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/ReduceTransformConfig",
"properties": {
"type": {
"const": "reducer",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/FilterTransformConfig",
"properties": {
"type": {
"const": "filter",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/RememberTransformConfig",
"properties": {
"type": {
"const": "remember",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/RecallTransformConfig",
"properties": {
"type": {
"const": "recall",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/MapTransformConfig",
"properties": {
"type": {
"const": "map",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/ScriptTransformConfig",
"properties": {
"type": {
"const": "script",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/DeadbandTransformConfig",
"properties": {
"type": {
"const": "deadband",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/ThrottleTransformConfig",
"properties": {
"type": {
"const": "throttle",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/PivotTransformConfig",
"properties": {
"type": {
"const": "pivot",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/DeriveTransformConfig",
"properties": {
"type": {
"const": "derive",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/RollingTransformConfig",
"properties": {
"type": {
"const": "rolling",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/SmoothTransformConfig",
"properties": {
"type": {
"const": "smooth",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/DetectTransformConfig",
"properties": {
"type": {
"const": "detect",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/ResampleTransformConfig",
"properties": {
"type": {
"const": "resample",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/FeaturesTransformConfig",
"properties": {
"type": {
"const": "features",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
}
],
"type": "object"
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "One pipeline. kayak merges all inputs into one stream. The stream goes\nthrough the transforms in sequence. Each batch that results goes to each\noutput.",
"properties": {
"id": {
"type": [
"string",
"null"
]
},
"inputs": {
"description": "The inputs of the pipeline. At least one input is required. Batches\narrive in the sequence in which the inputs make them. There is no\nsequence between two different inputs.",
"items": {
"$ref": "#/$defs/InputConfig"
},
"type": "array"
},
"outputs": {
"default": [],
"description": "The outputs. Optional. A pipeline that only sends to downstream\npipelines needs no output.",
"items": {
"$ref": "#/$defs/OutputConfig"
},
"type": "array"
},
"state": {
"anyOf": [
{
"$ref": "#/$defs/PipelineState"
},
{
"type": "null"
}
],
"description": "The state bucket of this pipeline, and the key of its messages. Required\nfor `remember`, `recall` and the streaming transforms that keep state,\nfor example `rolling`. These transforms fail to build without it."
},
"transforms": {
"default": [],
"description": "The transforms, in sequence. Optional. A pipeline that only moves\nmessages needs no transform.",
"items": {
"$ref": "#/$defs/TransformConfig"
},
"type": "array"
}
},
"required": [
"inputs"
],
"title": "Config",
"type": "object"
}Connections
All connections in the connections file, as an object from name to connection. kayak writes the names in alphabetical order.
schema
json
{
"$defs": {
"ClickhouseConnection": {
"description": "A ClickHouse server and the user that kayak connects as. kayak uses the\nHTTP interface of the server. The output or the input sets the table.",
"properties": {
"allow_http": {
"description": "Permit an `http://` url with no TLS. The default is false. The\ncredentials go with every request, as plain text. Use it only for a\nlocal server, for example the server in `docker-compose.yaml`.",
"type": [
"boolean",
"null"
]
},
"database": {
"description": "The database to use. The database must exist. The output makes tables.\nIt does not make databases.",
"type": "string"
},
"password": {
"$ref": "#/$defs/Secret",
"description": "The password of the user. Use a `${NAME}` secret reference for this\nvalue."
},
"url": {
"description": "The url of the HTTP interface, for example `http://localhost:8123` for\nthe server in `docker-compose.yaml`, or `https://<host>:8443` for\nClickHouse Cloud.",
"type": "string"
},
"user": {
"description": "The user to connect as.",
"type": "string"
}
},
"required": [
"url",
"database",
"user",
"password"
],
"title": "clickhouse",
"type": "object"
},
"ConnectionKind": {
"description": "The types of system that a connection can describe. The `type` field\nselects the type.\n\nInputs and outputs use the same connection types. For example, a kafka\ninput consumes from a `kafka` connection, and a kafka output publishes to\nit.",
"oneOf": [
{
"$ref": "#/$defs/KafkaConnection",
"properties": {
"type": {
"const": "kafka",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/NatsConnection",
"properties": {
"type": {
"const": "nats",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/PostgresConnection",
"properties": {
"type": {
"const": "postgres",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/ClickhouseConnection",
"properties": {
"type": {
"const": "clickhouse",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/FileConnection",
"properties": {
"type": {
"const": "file",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/S3Connection",
"properties": {
"type": {
"const": "s3",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/MqttConnection",
"properties": {
"type": {
"const": "mqtt",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/RedisConnection",
"properties": {
"type": {
"const": "redis",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/OpcuaConnection",
"properties": {
"type": {
"const": "opcua",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/InduConnection",
"properties": {
"type": {
"const": "indu",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/TidepoolConnection",
"properties": {
"type": {
"const": "tidepool",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
}
]
},
"FileConnection": {
"description": "A directory on the filesystem of the server. A `file` output writes under\nit, to a `path` relative to this directory.\n\nThe directory must be inside the `--data-dir` of the server. kayak checks\nthis when it builds the output. A server started without `--data-dir` has\nno file output.",
"properties": {
"root": {
"description": "The directory that file outputs write under, for example\n`./out/events`. It must be inside the `--data-dir` of the server. If the\ndirectory does not exist, kayak makes it.",
"type": "string"
}
},
"required": [
"root"
],
"title": "file",
"type": "object"
},
"InduConnection": {
"description": "An Indu Cloud deployment: the API and ingest endpoints, and the API key.\n\nThe `indu` output and the `indu` input use the same connection. The output\nwrites streams through `/ingest/v1/streams`. The input reads sensors and\nstreams through `/api/v1`. Make the key in Indu, on the `/keys` page or with\n`indud apps register --kind kayak`. Give the key a role in Indu.",
"properties": {
"api_key": {
"$ref": "#/$defs/Secret",
"description": "The API key (`indu.ak.…`), as a `${NAME}` secret reference."
},
"ingest_url": {
"description": "The origin of `/ingest/v1/…` when it is not under `url`, for example\n`https://ingest.acme.indu.cloud`. A single-server installation serves\ningest on its own host.",
"type": [
"string",
"null"
]
},
"url": {
"description": "The origin of the deployment, for example `https://app.acme.indu.cloud`.\nThe ingest endpoint is `/ingest/v1/…` under this url, unless you set\n`ingest_url`.",
"type": "string"
}
},
"required": [
"url",
"api_key"
],
"title": "indu",
"type": "object"
},
"KafkaConnection": {
"description": "A kafka cluster.",
"properties": {
"brokers": {
"$ref": "#/$defs/Secret",
"description": "The brokers, as a list with commas, for example `localhost:9092`. You\ncan use `${NAME}` secret references."
}
},
"required": [
"brokers"
],
"title": "kafka",
"type": "object"
},
"MqttConnection": {
"description": "An mqtt broker.\n\nThe connection uses plain TCP. TLS is not available.",
"properties": {
"host": {
"description": "The hostname of the broker, for example `localhost`.",
"type": "string"
},
"password": {
"anyOf": [
{
"$ref": "#/$defs/Secret"
},
{
"type": "null"
}
],
"description": "The password of the user. Use a `${NAME}` secret reference for this\nvalue."
},
"port": {
"description": "The port of the broker. The default is 1883.",
"format": "uint16",
"maximum": 65535,
"minimum": 0,
"type": [
"integer",
"null"
]
},
"username": {
"anyOf": [
{
"$ref": "#/$defs/Secret"
},
{
"type": "null"
}
],
"description": "The username, if the broker requires one. Set `username` and\n`password` together, or set neither."
}
},
"required": [
"host"
],
"title": "mqtt",
"type": "object"
},
"NatsConnection": {
"description": "A nats server, or a cluster of nats servers.",
"properties": {
"urls": {
"$ref": "#/$defs/Secret",
"description": "The url of the server, for example `nats://localhost:4222`. You can use\n`${NAME}` secret references."
}
},
"required": [
"urls"
],
"title": "nats",
"type": "object"
},
"OpcuaConnection": {
"description": "An OPC UA server. The connection holds the endpoint. The `opcua` input\nselects the nodes to read.\n\n**The session is not signed and not encrypted.** kayak connects with the\nsecurity policy `None`, as an anonymous user or with a username and\npassword. The credentials go over the network as plain text. Use this\nconnection only on a network that you trust.\n\nWhen a session opens, the OPC UA client writes two errors about a missing\napplication instance certificate to the log. This is not a fault. A session\nwith no encryption needs no certificate.",
"properties": {
"endpoint": {
"$ref": "#/$defs/Secret",
"description": "The url of the endpoint, for example `opc.tcp://localhost:50000`. You\ncan use `${NAME}` secret references.\n\nkayak connects directly to this url. It does not ask the server for its\nlist of endpoints first. Thus, a server behind docker, NAT or a load\nbalancer works when this url is correct."
},
"password": {
"anyOf": [
{
"$ref": "#/$defs/Secret"
},
{
"type": "null"
}
],
"description": "The password of the user. Use a `${NAME}` secret reference for this\nvalue."
},
"username": {
"anyOf": [
{
"$ref": "#/$defs/Secret"
},
{
"type": "null"
}
],
"description": "The username, if the server requires one. Set `username` and\n`password` together, or set neither. Without them, the session is\nanonymous."
}
},
"required": [
"endpoint"
],
"title": "opcua",
"type": "object"
},
"PostgresConnection": {
"description": "A postgres database and the role that kayak connects as. The output or\nthe input sets the table.",
"properties": {
"database": {
"description": "The database to connect to.",
"type": "string"
},
"host": {
"description": "The hostname of the server, for example `localhost`.",
"type": "string"
},
"password": {
"$ref": "#/$defs/Secret",
"description": "The password of the role. Use a `${NAME}` secret reference for this\nvalue."
},
"port": {
"description": "The port of the server. The default is 5432.",
"format": "uint16",
"maximum": 65535,
"minimum": 0,
"type": [
"integer",
"null"
]
},
"user": {
"description": "The role to connect as.",
"type": "string"
}
},
"required": [
"host",
"database",
"user",
"password"
],
"title": "postgres",
"type": "object"
},
"RedisConnection": {
"description": "A redis server, or a server that uses the same protocol.\n\nkayak uses the pub/sub commands `SUBSCRIBE` and `PUBLISH`. It does not use\nthe key-value store. Thus, a redis input has the same delivery guarantees\nas a nats input.",
"properties": {
"url": {
"$ref": "#/$defs/Secret",
"description": "The url of the server, for example `redis://localhost:6379` or\n`redis://:${REDIS_PASSWORD}@localhost:6379/0`. You can use `${NAME}`\nsecret references."
}
},
"required": [
"url"
],
"title": "redis",
"type": "object"
},
"S3Connection": {
"description": "A bucket on an S3-compatible object store, and its credentials. An `s3`\noutput writes under a prefix in the bucket.\n\nThere is no limit like `--data-dir` for a bucket. The credentials set what\nkayak can write. Give kayak a key that can write only to this bucket.\n\nSet `endpoint` for rustfs, minio or another S3-compatible server. Without\n`endpoint`, kayak uses AWS S3 in `region`.",
"properties": {
"access_key_id": {
"$ref": "#/$defs/Secret",
"description": "The access key id. Use a `${NAME}` secret reference for this value."
},
"allow_http": {
"description": "Permit an `http://` endpoint with no TLS. The default is false. The\ncredentials then go over the network as plain text. Use it only for a\nlocal server, for example the rustfs in `docker-compose.yaml`.",
"type": [
"boolean",
"null"
]
},
"bucket": {
"description": "The bucket to write to. The bucket must exist. The output makes\nobjects. It does not make buckets.",
"type": "string"
},
"endpoint": {
"description": "The url of an S3-compatible server, for example `http://localhost:9000`\nfor the rustfs in `docker-compose.yaml`. Leave it out to use AWS S3 in\n`region`.",
"type": [
"string",
"null"
]
},
"region": {
"description": "The region of the bucket. The default is `us-east-1`. S3-compatible\nservers with no regions accept this value.",
"type": [
"string",
"null"
]
},
"secret_access_key": {
"$ref": "#/$defs/Secret",
"description": "The secret access key. Use a `${NAME}` secret reference for this value."
}
},
"required": [
"bucket",
"access_key_id",
"secret_access_key"
],
"title": "s3",
"type": "object"
},
"Secret": {
"description": "A text value that can refer to secrets with `${NAME}` references.\n\nThe value is an ordinary JSON string. When kayak builds the pipeline, it\nreplaces each `${NAME}` reference with the value from the secret store of\nthe server:\n\n```json\n{ \"type\": \"nats\", \"urls\": \"nats://app:${NATS_PASSWORD}@broker:4222\" }\n```\n\nThe config keeps only the reference, never the value. Thus you can commit\nthe file, and `GET /api/pipelines` does not show the secret.\n\nkayak uses a value with no `${...}` reference as it is.",
"type": "string"
},
"TidepoolConnection": {
"description": "A Tidepool server and its ingest token. The `tidepool` output sets the\ntable. You declare the tables in the Tidepool project. kayak does not make\nthem.",
"properties": {
"allow_http": {
"description": "Permit an `http://` url with no TLS when `token` is set. The default is\nfalse. The token goes with every batch, as plain text. Without a token,\nan `http://` url is always permitted.",
"type": [
"boolean",
"null"
]
},
"token": {
"anyOf": [
{
"$ref": "#/$defs/Secret"
},
{
"type": "null"
}
],
"description": "The ingest token, as a `${NAME}` secret reference. Use the\n`TIDEPOOL_INGEST_TOKEN` of the server, or its admin token. Leave it out\nfor a server with open ingest."
},
"url": {
"description": "The url of the server, for example `http://localhost:7070`.",
"type": "string"
}
},
"required": [
"url"
],
"title": "tidepool",
"type": "object"
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema",
"additionalProperties": {
"$ref": "#/$defs/ConnectionKind"
},
"description": "All connections in the connections file, as an object from name to\nconnection. kayak writes the names in alphabetical order.",
"title": "Connections",
"type": "object"
}CreateConnectionRequest
The body of POST /api/connections: the name in id, and the fields of the connection beside it. The body has the same shape as one entry in the connections file.
schema
json
{
"$defs": {
"ClickhouseConnection": {
"description": "A ClickHouse server and the user that kayak connects as. kayak uses the\nHTTP interface of the server. The output or the input sets the table.",
"properties": {
"allow_http": {
"description": "Permit an `http://` url with no TLS. The default is false. The\ncredentials go with every request, as plain text. Use it only for a\nlocal server, for example the server in `docker-compose.yaml`.",
"type": [
"boolean",
"null"
]
},
"database": {
"description": "The database to use. The database must exist. The output makes tables.\nIt does not make databases.",
"type": "string"
},
"password": {
"$ref": "#/$defs/Secret",
"description": "The password of the user. Use a `${NAME}` secret reference for this\nvalue."
},
"url": {
"description": "The url of the HTTP interface, for example `http://localhost:8123` for\nthe server in `docker-compose.yaml`, or `https://<host>:8443` for\nClickHouse Cloud.",
"type": "string"
},
"user": {
"description": "The user to connect as.",
"type": "string"
}
},
"required": [
"url",
"database",
"user",
"password"
],
"title": "clickhouse",
"type": "object"
},
"FileConnection": {
"description": "A directory on the filesystem of the server. A `file` output writes under\nit, to a `path` relative to this directory.\n\nThe directory must be inside the `--data-dir` of the server. kayak checks\nthis when it builds the output. A server started without `--data-dir` has\nno file output.",
"properties": {
"root": {
"description": "The directory that file outputs write under, for example\n`./out/events`. It must be inside the `--data-dir` of the server. If the\ndirectory does not exist, kayak makes it.",
"type": "string"
}
},
"required": [
"root"
],
"title": "file",
"type": "object"
},
"InduConnection": {
"description": "An Indu Cloud deployment: the API and ingest endpoints, and the API key.\n\nThe `indu` output and the `indu` input use the same connection. The output\nwrites streams through `/ingest/v1/streams`. The input reads sensors and\nstreams through `/api/v1`. Make the key in Indu, on the `/keys` page or with\n`indud apps register --kind kayak`. Give the key a role in Indu.",
"properties": {
"api_key": {
"$ref": "#/$defs/Secret",
"description": "The API key (`indu.ak.…`), as a `${NAME}` secret reference."
},
"ingest_url": {
"description": "The origin of `/ingest/v1/…` when it is not under `url`, for example\n`https://ingest.acme.indu.cloud`. A single-server installation serves\ningest on its own host.",
"type": [
"string",
"null"
]
},
"url": {
"description": "The origin of the deployment, for example `https://app.acme.indu.cloud`.\nThe ingest endpoint is `/ingest/v1/…` under this url, unless you set\n`ingest_url`.",
"type": "string"
}
},
"required": [
"url",
"api_key"
],
"title": "indu",
"type": "object"
},
"KafkaConnection": {
"description": "A kafka cluster.",
"properties": {
"brokers": {
"$ref": "#/$defs/Secret",
"description": "The brokers, as a list with commas, for example `localhost:9092`. You\ncan use `${NAME}` secret references."
}
},
"required": [
"brokers"
],
"title": "kafka",
"type": "object"
},
"MqttConnection": {
"description": "An mqtt broker.\n\nThe connection uses plain TCP. TLS is not available.",
"properties": {
"host": {
"description": "The hostname of the broker, for example `localhost`.",
"type": "string"
},
"password": {
"anyOf": [
{
"$ref": "#/$defs/Secret"
},
{
"type": "null"
}
],
"description": "The password of the user. Use a `${NAME}` secret reference for this\nvalue."
},
"port": {
"description": "The port of the broker. The default is 1883.",
"format": "uint16",
"maximum": 65535,
"minimum": 0,
"type": [
"integer",
"null"
]
},
"username": {
"anyOf": [
{
"$ref": "#/$defs/Secret"
},
{
"type": "null"
}
],
"description": "The username, if the broker requires one. Set `username` and\n`password` together, or set neither."
}
},
"required": [
"host"
],
"title": "mqtt",
"type": "object"
},
"NatsConnection": {
"description": "A nats server, or a cluster of nats servers.",
"properties": {
"urls": {
"$ref": "#/$defs/Secret",
"description": "The url of the server, for example `nats://localhost:4222`. You can use\n`${NAME}` secret references."
}
},
"required": [
"urls"
],
"title": "nats",
"type": "object"
},
"OpcuaConnection": {
"description": "An OPC UA server. The connection holds the endpoint. The `opcua` input\nselects the nodes to read.\n\n**The session is not signed and not encrypted.** kayak connects with the\nsecurity policy `None`, as an anonymous user or with a username and\npassword. The credentials go over the network as plain text. Use this\nconnection only on a network that you trust.\n\nWhen a session opens, the OPC UA client writes two errors about a missing\napplication instance certificate to the log. This is not a fault. A session\nwith no encryption needs no certificate.",
"properties": {
"endpoint": {
"$ref": "#/$defs/Secret",
"description": "The url of the endpoint, for example `opc.tcp://localhost:50000`. You\ncan use `${NAME}` secret references.\n\nkayak connects directly to this url. It does not ask the server for its\nlist of endpoints first. Thus, a server behind docker, NAT or a load\nbalancer works when this url is correct."
},
"password": {
"anyOf": [
{
"$ref": "#/$defs/Secret"
},
{
"type": "null"
}
],
"description": "The password of the user. Use a `${NAME}` secret reference for this\nvalue."
},
"username": {
"anyOf": [
{
"$ref": "#/$defs/Secret"
},
{
"type": "null"
}
],
"description": "The username, if the server requires one. Set `username` and\n`password` together, or set neither. Without them, the session is\nanonymous."
}
},
"required": [
"endpoint"
],
"title": "opcua",
"type": "object"
},
"PostgresConnection": {
"description": "A postgres database and the role that kayak connects as. The output or\nthe input sets the table.",
"properties": {
"database": {
"description": "The database to connect to.",
"type": "string"
},
"host": {
"description": "The hostname of the server, for example `localhost`.",
"type": "string"
},
"password": {
"$ref": "#/$defs/Secret",
"description": "The password of the role. Use a `${NAME}` secret reference for this\nvalue."
},
"port": {
"description": "The port of the server. The default is 5432.",
"format": "uint16",
"maximum": 65535,
"minimum": 0,
"type": [
"integer",
"null"
]
},
"user": {
"description": "The role to connect as.",
"type": "string"
}
},
"required": [
"host",
"database",
"user",
"password"
],
"title": "postgres",
"type": "object"
},
"RedisConnection": {
"description": "A redis server, or a server that uses the same protocol.\n\nkayak uses the pub/sub commands `SUBSCRIBE` and `PUBLISH`. It does not use\nthe key-value store. Thus, a redis input has the same delivery guarantees\nas a nats input.",
"properties": {
"url": {
"$ref": "#/$defs/Secret",
"description": "The url of the server, for example `redis://localhost:6379` or\n`redis://:${REDIS_PASSWORD}@localhost:6379/0`. You can use `${NAME}`\nsecret references."
}
},
"required": [
"url"
],
"title": "redis",
"type": "object"
},
"S3Connection": {
"description": "A bucket on an S3-compatible object store, and its credentials. An `s3`\noutput writes under a prefix in the bucket.\n\nThere is no limit like `--data-dir` for a bucket. The credentials set what\nkayak can write. Give kayak a key that can write only to this bucket.\n\nSet `endpoint` for rustfs, minio or another S3-compatible server. Without\n`endpoint`, kayak uses AWS S3 in `region`.",
"properties": {
"access_key_id": {
"$ref": "#/$defs/Secret",
"description": "The access key id. Use a `${NAME}` secret reference for this value."
},
"allow_http": {
"description": "Permit an `http://` endpoint with no TLS. The default is false. The\ncredentials then go over the network as plain text. Use it only for a\nlocal server, for example the rustfs in `docker-compose.yaml`.",
"type": [
"boolean",
"null"
]
},
"bucket": {
"description": "The bucket to write to. The bucket must exist. The output makes\nobjects. It does not make buckets.",
"type": "string"
},
"endpoint": {
"description": "The url of an S3-compatible server, for example `http://localhost:9000`\nfor the rustfs in `docker-compose.yaml`. Leave it out to use AWS S3 in\n`region`.",
"type": [
"string",
"null"
]
},
"region": {
"description": "The region of the bucket. The default is `us-east-1`. S3-compatible\nservers with no regions accept this value.",
"type": [
"string",
"null"
]
},
"secret_access_key": {
"$ref": "#/$defs/Secret",
"description": "The secret access key. Use a `${NAME}` secret reference for this value."
}
},
"required": [
"bucket",
"access_key_id",
"secret_access_key"
],
"title": "s3",
"type": "object"
},
"Secret": {
"description": "A text value that can refer to secrets with `${NAME}` references.\n\nThe value is an ordinary JSON string. When kayak builds the pipeline, it\nreplaces each `${NAME}` reference with the value from the secret store of\nthe server:\n\n```json\n{ \"type\": \"nats\", \"urls\": \"nats://app:${NATS_PASSWORD}@broker:4222\" }\n```\n\nThe config keeps only the reference, never the value. Thus you can commit\nthe file, and `GET /api/pipelines` does not show the secret.\n\nkayak uses a value with no `${...}` reference as it is.",
"type": "string"
},
"TidepoolConnection": {
"description": "A Tidepool server and its ingest token. The `tidepool` output sets the\ntable. You declare the tables in the Tidepool project. kayak does not make\nthem.",
"properties": {
"allow_http": {
"description": "Permit an `http://` url with no TLS when `token` is set. The default is\nfalse. The token goes with every batch, as plain text. Without a token,\nan `http://` url is always permitted.",
"type": [
"boolean",
"null"
]
},
"token": {
"anyOf": [
{
"$ref": "#/$defs/Secret"
},
{
"type": "null"
}
],
"description": "The ingest token, as a `${NAME}` secret reference. Use the\n`TIDEPOOL_INGEST_TOKEN` of the server, or its admin token. Leave it out\nfor a server with open ingest."
},
"url": {
"description": "The url of the server, for example `http://localhost:7070`.",
"type": "string"
}
},
"required": [
"url"
],
"title": "tidepool",
"type": "object"
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "The body of `POST /api/connections`: the name in `id`, and the fields of\nthe connection beside it. The body has the same shape as one entry in the\nconnections file.",
"oneOf": [
{
"$ref": "#/$defs/KafkaConnection",
"properties": {
"type": {
"const": "kafka",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/NatsConnection",
"properties": {
"type": {
"const": "nats",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/PostgresConnection",
"properties": {
"type": {
"const": "postgres",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/ClickhouseConnection",
"properties": {
"type": {
"const": "clickhouse",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/FileConnection",
"properties": {
"type": {
"const": "file",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/S3Connection",
"properties": {
"type": {
"const": "s3",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/MqttConnection",
"properties": {
"type": {
"const": "mqtt",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/RedisConnection",
"properties": {
"type": {
"const": "redis",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/OpcuaConnection",
"properties": {
"type": {
"const": "opcua",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/InduConnection",
"properties": {
"type": {
"const": "indu",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/TidepoolConnection",
"properties": {
"type": {
"const": "tidepool",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
}
],
"properties": {
"id": {
"type": "string"
}
},
"required": [
"id"
],
"title": "CreateConnectionRequest",
"type": "object"
}DryRunRequest
The body of POST /api/scripts/dry-run.
The dry run uses the same runner as the script transform, with the same settings.
schema
json
{
"$defs": {
"ScriptScope": {
"description": "Whether the script gets one message or the full batch.\n\n`message` is the default. The operation limit then applies to each message,\nand the batch keeps its structure.\n\nUse `batch` for work on the full batch. For example, remove duplicates in\nthe batch, or calculate a value that `reduce` has no function for.",
"oneOf": [
{
"const": "message",
"description": "The script runs one time for each message, with the message in `msg`.",
"type": "string"
},
{
"const": "batch",
"description": "The script runs one time for each batch, with the messages in `batch`\nas an array. When you emit an array, the transform sends a batch of\nthose messages.",
"type": "string"
}
]
},
"ScriptSource": {
"description": "The source of the script text: `inline` or `file`.\n\nUse `inline` to send a script through the HTTP API. In a YAML config, an\ninline script is a literal block. Use `file` to keep the script in its own\nfile, where an editor, a formatter and a test can use it. A save keeps the\n`file` reference in the config. It does not copy the script into the\nconfig.",
"oneOf": [
{
"description": "The script text, in the config. Use a YAML config for an inline script.\nYAML keeps the script as a literal block. JSON must escape each newline.",
"properties": {
"code": {
"description": "The rhai source.",
"type": "string",
"x-script": "rhai"
},
"type": {
"const": "inline",
"type": "string"
}
},
"required": [
"type",
"code"
],
"type": "object"
},
{
"description": "A path to a `.rhai` file, relative to the directory of the config file.\n\nkayak reads the file and its imports when it builds the pipeline. After\nyou change the file, do a revert to use the change. A server with no\nconfig file does not accept a `file` source. Inline scripts work on that\nserver, but their imports do not.",
"properties": {
"path": {
"description": "The path, relative to the directory of the config file. The path\nmust stay in that directory.",
"type": "string"
},
"type": {
"const": "file",
"type": "string"
}
},
"required": [
"type",
"path"
],
"type": "object"
}
]
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "The body of `POST /api/scripts/dry-run`.\n\nThe dry run uses the same runner as the `script` transform, with the same\nsettings.",
"properties": {
"max_operations": {
"description": "The operation limit for this run. The default is the same as for the\ntransform.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"messages": {
"default": [],
"description": "The messages to run the script over, as one batch. An empty list is\npermitted. Use it to check that the script compiles.",
"items": true,
"type": "array"
},
"scope": {
"$ref": "#/$defs/ScriptScope",
"default": "message",
"description": "Whether the script gets one message at a time or the full batch. The\ndefault is `message`."
},
"source": {
"$ref": "#/$defs/ScriptSource",
"description": "The script to run, inline or in a file, as in the transform."
},
"state": {
"additionalProperties": {
"additionalProperties": true,
"type": "object"
},
"description": "The initial contents of the state bucket of the run, by the key that a\nscript uses with `recall`.\n\nA dry run **never uses a live bucket**. It gets a private bucket with\nthese contents. kayak discards the bucket after the run.",
"type": "object"
}
},
"required": [
"source"
],
"title": "DryRunRequest",
"type": "object"
}DryRunResponse
The response of a script dry run. The outcome field selects the variant.
A script that does not compile gives a 200 with the failed outcome, not a 400.
schema
json
{
"$defs": {
"DryRunStage": {
"description": "When a script failure occurred.",
"oneOf": [
{
"const": "compile",
"description": "The script does not parse. A pipeline with this script does not start.",
"type": "string"
},
{
"const": "runtime",
"description": "The script parsed, and this run of it failed. A pipeline with this\nscript starts, and fails the batches that cause the error.",
"type": "string"
}
]
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "The response of a script dry run. The `outcome` field selects the variant.\n\nA script that does not compile gives a **200 with the `failed` outcome**,\nnot a 400.",
"oneOf": [
{
"description": "The script ran. This includes a script that emitted nothing.",
"properties": {
"batches": {
"description": "The batches that the script emitted, in order. In `message` scope,\nthere is a maximum of one. In `batch` scope, there is one for each\n`emit`.",
"items": {
"items": true,
"type": "array"
},
"type": "array"
},
"outcome": {
"const": "emitted",
"type": "string"
},
"state": {
"additionalProperties": {
"additionalProperties": true,
"type": "object"
},
"description": "The contents of the private bucket after the run. kayak discards\nthe bucket after the response.",
"type": "object"
},
"warnings": {
"description": "The different texts that the script gave to `warn()`.",
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"outcome",
"batches"
],
"type": "object"
},
{
"description": "The script did not compile, or a run of it failed.",
"properties": {
"column": {
"description": "The column, from 1. Absent when `line` is absent.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"line": {
"description": "The line, from 1. Absent when the failure has no line, for example\nwhen the operation limit is reached, or when a built-in function\ngives an error.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"message": {
"description": "The error message from rhai, without the position. The position is\nin `line` and `column`.",
"type": "string"
},
"outcome": {
"const": "failed",
"type": "string"
},
"stage": {
"$ref": "#/$defs/DryRunStage",
"description": "Whether the failure occurred when the script compiled or when it\nran."
}
},
"required": [
"outcome",
"stage",
"message"
],
"type": "object"
}
],
"title": "DryRunResponse"
}IngestRequest
The body of POST /api/pipelines/{pipeline_id}/messages: one message, or an array of messages. An array is always many messages, not one message.
schema
json
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"anyOf": [
{
"description": "Many messages, sent as one batch.",
"items": true,
"type": "array"
},
{
"description": "One message, sent as a batch of one."
}
],
"description": "The body of `POST /api/pipelines/{pipeline_id}/messages`: one message, or an\narray of messages. An array is always many messages, not one message.",
"title": "IngestRequest"
}IngestResponse
The response to a post: the number of messages that the pipeline accepted.
Accepted means that the batch is in the queue of the run loop. It does not mean that the outputs wrote the messages.
schema
json
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "The response to a post: the number of messages that the pipeline accepted.\n\nAccepted means that the batch is in the queue of the run loop. It does not\nmean that the outputs wrote the messages.",
"properties": {
"accepted": {
"format": "uint",
"minimum": 0,
"type": "integer"
}
},
"required": [
"accepted"
],
"title": "IngestResponse",
"type": "object"
}LayoutFile
The whole file: every pipeline someone has placed by hand, and every edge whose route they have adjusted.
Absent ids are the normal case, not a gap to be filled — the canvas lays those out itself. A pipeline is added here the first time it is dragged and removed when the arrangement is reset.
schema
json
{
"$defs": {
"EdgeLayout": {
"description": "One edge's routing, where the automatic answer wasn't the readable one.\n\nAn edge between two cards runs out of one, along a channel between them, and\ninto the other. `offset` moves that channel off the half-way line — pulled\ntowards one card or the other — which is how a route that the automatic\nseparation put somewhere unhelpful gets moved. The two ports say where on\ntheir cards the edge attaches.\n\n`None` is not \"zero\": it means nobody has touched this channel, so the canvas\nis free to place it — and it does, spreading edges that would otherwise lie\non top of each other onto lines of their own. A stored `Some(0.0)` is a\ndeliberate \"on the half-way line, whatever else is there\", which is why this\nis an option rather than a number with a default.\n\nThe offset is *relative* rather than an absolute coordinate, because cards\nmove: storing \"20 above the middle\" keeps the adjustment meaningful after\neither end is dragged, where storing a y would eventually put the channel\noutside the gap it was meant to sit in.\n\nListed rather than keyed by a joined id, so nothing has to be escaped and no\nid character is special.",
"properties": {
"from": {
"type": "string"
},
"from_port": {
"anyOf": [
{
"$ref": "#/$defs/PortLayout"
},
{
"type": "null"
}
]
},
"offset": {
"format": "double",
"type": [
"number",
"null"
]
},
"to": {
"type": "string"
},
"to_port": {
"anyOf": [
{
"$ref": "#/$defs/PortLayout"
},
{
"type": "null"
}
]
}
},
"required": [
"from",
"to"
],
"type": "object"
},
"PipelineLayout": {
"description": "One card's place on the canvas, in surface coordinates (the same space the\nautomatic layout works in — pixels at zoom 1, origin at the top-left of the\ngraph).\n\n`height` is optional because the two ways a card gets its height are\ndifferent things: normally it is *measured* from the content it renders, and\nonly an explicit resize pins it. `None` means \"however tall it needs to be\",\nwhich is what you want a card to go back to when its config grows.",
"properties": {
"height": {
"format": "double",
"type": [
"number",
"null"
]
},
"width": {
"format": "double",
"type": "number"
},
"x": {
"format": "double",
"type": "number"
},
"y": {
"format": "double",
"type": "number"
}
},
"required": [
"x",
"y",
"width"
],
"type": "object"
},
"PortLayout": {
"description": "Where an edge attaches to a card, once someone has said so by hand.\n\n`along` is measured from the start of the face — its left end for a top or\nbottom face, its top end for a left or right one — rather than as a fraction\nof it. A distance is what the drag meant: \"attach a card's width in from the\ncorner\" stays put when the card is made taller, where a fraction would slide.\n\n`side` is carried because the automatic router can change its mind: move a\ncard and an edge that left by the bottom may now leave by the side, and a\nposition measured along the old face means nothing on the new one. When they\ndisagree the stored position is ignored and the edge goes back to being\nplaced automatically, which is self-healing and needs no cleanup pass.",
"properties": {
"along": {
"format": "double",
"type": "number"
},
"side": {
"$ref": "#/$defs/Side"
}
},
"required": [
"side",
"along"
],
"type": "object"
},
"Side": {
"description": "A face of a card. Which one an edge uses is worked out from where the two\ncards sit and is never stored — but *where along it* the edge attaches can\nbe, and a stored position only means anything together with the face it was\nmeasured on.",
"enum": [
"top",
"right",
"bottom",
"left"
],
"type": "string"
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "The whole file: every pipeline someone has placed by hand, and every edge whose\nroute they have adjusted.\n\nAbsent ids are the normal case, not a gap to be filled — the canvas lays\nthose out itself. A pipeline is added here the first time it is dragged and\nremoved when the arrangement is reset.",
"properties": {
"edges": {
"description": "Kept sorted by `(from, to)` — see [`LayoutFile::set_edge_offset`] — so\nthe file stays diffable.",
"items": {
"$ref": "#/$defs/EdgeLayout"
},
"type": "array"
},
"pipelines": {
"additionalProperties": {
"$ref": "#/$defs/PipelineLayout"
},
"default": {},
"type": "object"
},
"version": {
"default": 1,
"format": "uint32",
"minimum": 0,
"type": "integer"
}
},
"title": "LayoutFile",
"type": "object"
}LoadedScript
The text that kayak compiled a running script transform from, and the modules that it imported.
This is the text from the build of the pipeline. If a file changed after the build, changed_on_disk is true.
schema
json
{
"$defs": {
"LoadedModule": {
"description": "One module that a script imported, as it was when kayak built the\npipeline.",
"properties": {
"changed_on_disk": {
"default": false,
"description": "True when the file is different from `code`, or cannot be read.",
"type": "boolean"
},
"code": {
"description": "The rhai source of the module.",
"type": "string"
},
"path": {
"description": "The file of the module, relative to the directory of the config file,\nwith the `.rhai` extension.",
"type": "string"
}
},
"required": [
"path",
"code"
],
"type": "object"
},
"ScriptScope": {
"description": "Whether the script gets one message or the full batch.\n\n`message` is the default. The operation limit then applies to each message,\nand the batch keeps its structure.\n\nUse `batch` for work on the full batch. For example, remove duplicates in\nthe batch, or calculate a value that `reduce` has no function for.",
"oneOf": [
{
"const": "message",
"description": "The script runs one time for each message, with the message in `msg`.",
"type": "string"
},
{
"const": "batch",
"description": "The script runs one time for each batch, with the messages in `batch`\nas an array. When you emit an array, the transform sends a batch of\nthose messages.",
"type": "string"
}
]
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "The text that kayak compiled a running `script` transform from, and the\nmodules that it imported.\n\nThis is the text from the build of the pipeline. If a file changed after\nthe build, `changed_on_disk` is true.",
"properties": {
"changed_on_disk": {
"default": false,
"description": "True when the file at `path` is different from `code`, or cannot be\nread. This means that the file changed after the build. Always false for\nan inline script. A revert uses the change.",
"type": "boolean"
},
"code": {
"description": "The rhai source that the transform compiled.",
"type": "string"
},
"modules": {
"default": [],
"description": "All modules that the script imported, directly or through another\nmodule, in the order of the first import. Empty for a script with no\nimports.",
"items": {
"$ref": "#/$defs/LoadedModule"
},
"type": "array"
},
"path": {
"description": "The file of the script, relative to the directory of the config file.\nAbsent for an inline script.",
"type": [
"string",
"null"
]
},
"scope": {
"$ref": "#/$defs/ScriptScope",
"description": "Whether the script runs for each message or for each batch."
}
},
"required": [
"scope",
"code"
],
"title": "LoadedScript",
"type": "object"
}LoginRequest
The body of POST /api/auth/login.
The password is the real password, not a ${NAME} reference. kayak does not store it, send it back or write it to the log.
schema
json
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "The body of `POST /api/auth/login`.\n\nThe password is the real password, not a `${NAME}` reference. kayak does\nnot store it, send it back or write it to the log.",
"properties": {
"password": {
"type": "string"
},
"username": {
"type": "string"
}
},
"required": [
"username",
"password"
],
"title": "LoginRequest",
"type": "object"
}OpenApiDocument
An OpenAPI 3.1 document. See https://spec.openapis.org/oas/v3.1.0
schema
json
{
"description": "An OpenAPI 3.1 document. See https://spec.openapis.org/oas/v3.1.0",
"properties": {
"components": {
"type": "object"
},
"info": {
"type": "object"
},
"openapi": {
"type": "string"
},
"paths": {
"type": "object"
}
},
"required": [
"openapi",
"paths"
],
"title": "OpenApiDocument",
"type": "object"
}PipelineDryRunRequest
Run a draft's transforms over some messages.
schema
json
{
"$defs": {
"Aggregation": {
"description": "One value to calculate for a group, and the field name for the result.",
"properties": {
"as": {
"description": "The field that contains the result in the sent message. Each\naggregation must have a different `as`. It must not be the same as a\n`group_by` field.",
"type": "string"
},
"field": {
"description": "The field to aggregate. Required for all functions except `count`.\nWithout a `field`, `count` counts the messages.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"function": {
"$ref": "#/$defs/ReduceFnKind",
"description": "How to combine the values."
}
},
"required": [
"function",
"as"
],
"title": "aggregation",
"type": "object"
},
"ArithmeticOperator": {
"description": "What an [`Mapping::Arithmetic`] does with its two operands.",
"oneOf": [
{
"const": "add",
"description": "Left + right.",
"type": "string"
},
{
"const": "subtract",
"description": "Left − right.",
"type": "string"
},
{
"const": "multiply",
"description": "Left × right.",
"type": "string"
},
{
"const": "divide",
"description": "Left ÷ right. A literal zero on the right is an error when the pipeline\nbuilds. For a field that holds zero, `on_zero` applies.",
"type": "string"
},
{
"const": "min",
"description": "The smaller of left and right. With a literal on one side, this is an\nupper limit.",
"type": "string"
},
{
"const": "max",
"description": "The larger of left and right. With a literal on one side, this is a\nlower limit.",
"type": "string"
}
]
},
"Band": {
"description": "The power in one frequency band, as a feature.",
"properties": {
"as": {
"description": "The field to write the power of the band to.",
"type": "string"
},
"high_hz": {
"description": "The upper limit of the band, in hertz. The band does not include this\nvalue. It must be more than `low_hz`.",
"format": "double",
"type": "number"
},
"low_hz": {
"description": "The lower limit of the band, in hertz. The band includes this value.",
"format": "double",
"type": "number"
}
},
"required": [
"low_hz",
"high_hz",
"as"
],
"title": "band",
"type": "object"
},
"BufferGateConfig": {
"description": "A condition on a state bucket that releases a `buffer` transform.\n\nThe buffer tests the conditions against the bucket entry as an object. The\nnames that `remember` wrote are its fields. `field` is a dotted path, as in\nall other transforms. All conditions must be true.\n\nThe gate applies to the full buffer. It does not test each kept message.\nWhen the gate opens, the buffer sends all kept messages.",
"properties": {
"bucket": {
"description": "The bucket to watch. The default is the bucket in the `state` of this\npipeline. A pipeline with no `state` must set it.",
"type": [
"string",
"null"
]
},
"conditions": {
"description": "The conditions that must all be true to release the buffer. At least\none condition is required.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"key": {
"description": "The key in the bucket to read. This is a literal key, not a field path.\nIf you do not set it, the gate reads the value for the full bucket.\n`remember` writes that value when the `state` of its pipeline has no\n`key`.",
"type": [
"string",
"null"
]
}
},
"required": [
"conditions"
],
"title": "buffer gate",
"type": "object"
},
"BufferTransformConfig": {
"description": "Keeps messages and sends them on when a trigger fires.\n\nThere are three triggers: a message count, a time, and a condition on a\nstate bucket. You can use them together. The first trigger that fires\nreleases the messages. A buffer with no trigger fails to build.\n\n`size` sends batches of exactly that number of messages. `seconds` and\n`until` send all messages that the buffer keeps, as one batch.\n\nThe `buffer` setting on an input is a different thing. It makes batches\nbefore the transforms. The `buffer` transform makes batches at its\nposition in the chain, for example after a `filter` or a `recall`.",
"properties": {
"max_messages": {
"description": "The maximum number of messages to keep. At this number, the buffer sends\nall kept messages and writes one warning to the log. Required if `size`\nis not set. Without a limit, a condition that is never true makes the\nbuffer use more and more memory.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"seconds": {
"description": "Send all kept messages this number of seconds after the first kept\nmessage. The time starts when the buffer keeps a message. An empty\nbuffer sends nothing.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"size": {
"description": "Send batches of exactly this number of messages, when each batch is\nfull.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"until": {
"anyOf": [
{
"$ref": "#/$defs/BufferGateConfig"
},
{
"type": "null"
}
],
"description": "Send all kept messages when a condition on a state bucket is true.\nBuckets are global, so a different pipeline can write the value that\nopens the gate."
}
},
"title": "buffer",
"type": "object"
},
"CastType": {
"description": "The type that a `cast` converts a value to.\n\nThese types are not the column types of the database outputs. There is no\n`bigint` and no `decimal`. Here, `json` parses a string that contains JSON.",
"oneOf": [
{
"const": "text",
"description": "A string. A number or a boolean is written as JSON writes it. An object\nor an array is an error.",
"type": "string"
},
{
"const": "integer",
"description": "A whole number. A string is parsed. A number with a fractional part is\nan error. The cast does not round.",
"type": "string"
},
{
"const": "float",
"description": "A number. A string is parsed.",
"type": "string"
},
{
"const": "boolean",
"description": "True or false. The cast accepts the strings `true` and `false` in\nuppercase or lowercase, and the numbers 1 and 0. Other values are an\nerror.",
"type": "string"
},
{
"const": "timestamp",
"description": "A timestamp, written as RFC 3339. A string is parsed and written again\nin one format. A number is read as **seconds** since the epoch, with\nfractions. The column mapping reads a number in the same way.",
"type": "string"
},
{
"const": "date",
"description": "A calendar date, written as `2026-08-10`. A string can be a date or a\nfull timestamp. From a timestamp, the cast uses the date.",
"type": "string"
},
{
"const": "uuid",
"description": "A UUID, in lowercase. The cast accepts only a string in the canonical\nform with hyphens.",
"type": "string"
},
{
"const": "json",
"description": "Parses the JSON that a string contains. The input must be a string. Use\nit for a payload that is encoded inside another payload.",
"type": "string"
}
]
},
"ConcatPart": {
"description": "One piece of a [`Mapping::Concat`].",
"oneOf": [
{
"description": "A value from the message. A string is used as it is. A number or a\nboolean is written as JSON writes it. An object or an array is an error.",
"properties": {
"field": {
"description": "The field to read.",
"type": "string"
},
"type": {
"const": "field",
"type": "string"
}
},
"required": [
"type",
"field"
],
"type": "object"
},
{
"description": "Literal text, for example a separator, a prefix or a suffix.",
"properties": {
"type": {
"const": "value",
"type": "string"
},
"value": {
"description": "The text.",
"type": "string"
}
},
"required": [
"type",
"value"
],
"type": "object"
}
]
},
"Condition": {
"description": "One test on a message.\n\n`filter`, `remember` and the gate of a `buffer` use conditions. A list of\nconditions means that all of them must match. There is no `or` and no\nnesting. Use `invert` on a `filter`, or `none_of`, for a negative test.",
"oneOf": [
{
"description": "Compares a field to a number. A message whose field is missing or is not\na number does not match.",
"properties": {
"field": {
"description": "The field to test, as a dotted path.",
"type": "string",
"x-message-field": true
},
"operator": {
"$ref": "#/$defs/NumericFilterOperatorKind",
"description": "How to compare the field."
},
"type": {
"const": "numeric",
"type": "string"
},
"value": {
"description": "The number to compare to.",
"format": "double",
"type": "number"
}
},
"required": [
"type",
"field",
"operator",
"value"
],
"type": "object"
},
{
"description": "Compares a field to a string. A message whose field is missing or is not\na string does not match.",
"properties": {
"field": {
"description": "The field to test, as a dotted path.",
"type": "string",
"x-message-field": true
},
"operator": {
"$ref": "#/$defs/StringFilterOperatorKind",
"description": "How to compare the field."
},
"type": {
"const": "string",
"type": "string"
},
"value": {
"description": "The string to compare to.",
"type": "string"
}
},
"required": [
"type",
"field",
"operator",
"value"
],
"type": "object"
},
{
"description": "Matches when the field is a string equal to one of `values`. A message\nwhose field is missing or is not a string does not match.",
"properties": {
"field": {
"description": "The field to test, as a dotted path.",
"type": "string",
"x-message-field": true
},
"type": {
"const": "one_of",
"type": "string"
},
"values": {
"description": "The strings that match.",
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"type",
"field",
"values"
],
"type": "object"
},
{
"description": "Matches when the field is a string equal to none of `values`. A message\nwhose field is missing or is not a string also does not match.",
"properties": {
"field": {
"description": "The field to test, as a dotted path.",
"type": "string",
"x-message-field": true
},
"type": {
"const": "none_of",
"type": "string"
},
"values": {
"description": "The strings that do not match.",
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"type",
"field",
"values"
],
"type": "object"
}
]
},
"DeadbandMode": {
"description": "The unit of the `delta` of a deadband: an amount, or a percentage of the\nlast value that passed.",
"oneOf": [
{
"const": "absolute",
"description": "`delta` is in the units of the field. For example, `0.5` is half a\ndegree.",
"type": "string"
},
{
"const": "percent",
"description": "`delta` is a percentage of the last value that passed. For example,\n`2` is two percent. When the last value is zero, every message passes.",
"type": "string"
}
]
},
"DeadbandTransformConfig": {
"description": "Drops a message when its field did not change sufficiently since the last\nmessage that passed. This is a filter that keeps state per key.\n\nThe first message for each key always passes. After that, a message passes\nin two cases:\n\n- `field` differs from the last value that passed by more than `delta`.\n- `max_seconds` went by since the last message passed.\n\nWith `flatline_seconds`, the transform also finds a stuck sensor. When the\nvalue does not change for that time, the next message passes with\n`stuck: true`. This occurs one time for each flat period.",
"properties": {
"delta": {
"description": "The change that a message needs to pass. The unit is set by `mode`.",
"format": "double",
"type": "number"
},
"field": {
"description": "The numeric field to compare.",
"type": "string",
"x-message-field": true
},
"flatline_seconds": {
"description": "After this number of seconds with no change, pass the next message with\n`stuck: true`. This occurs one time for each flat period.",
"format": "double",
"type": [
"number",
"null"
]
},
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out for\none series.",
"items": {
"type": "string"
},
"type": "array"
},
"max_seconds": {
"description": "Pass a message when this number of seconds went by since the last\nmessage passed, also if the value did not change.",
"format": "double",
"type": [
"number",
"null"
]
},
"mode": {
"$ref": "#/$defs/DeadbandMode",
"description": "The unit of `delta`. The default is `absolute`."
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have the field or a group\nfield. The default is `error`."
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"time": {
"description": "The field that holds the time of each message, as an RFC 3339 string\nor as milliseconds since the epoch. Leave it out to use the arrival\ntime.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"field",
"delta"
],
"title": "deadband",
"type": "object"
},
"Derivation": {
"description": "One derived value and the field it is written to.",
"properties": {
"as": {
"description": "The field to write the result to.",
"type": "string"
},
"field": {
"description": "The numeric field to read.",
"type": "string",
"x-message-field": true
},
"function": {
"$ref": "#/$defs/DeriveFnKind",
"description": "The calculation to do."
},
"wrap_at": {
"description": "For `counter`: the value at which the counter goes back to zero. With\nit, the transform reads a decrease as a wrap. Without it, the\ntransform reads a decrease as a reset.",
"format": "double",
"type": [
"number",
"null"
]
}
},
"required": [
"function",
"field",
"as"
],
"title": "derivation",
"type": "object"
},
"DeriveFnKind": {
"description": "How the transform calculates a value from this message and the previous\nmessage.",
"oneOf": [
{
"const": "rate",
"description": "The change per second since the previous message, from the `time`\nfield. The value is `null` for the first message and when no time went\nby.",
"type": "string"
},
{
"const": "delta",
"description": "The change since the previous message. The value is `null` for the\nfirst message.",
"type": "string"
},
{
"const": "cumsum",
"description": "The running total of the field, from the first message.",
"type": "string"
},
{
"const": "counter",
"description": "The running total of the increases. Use it for a counter that resets\nor wraps. When the value decreases and `wrap_at` is set, the transform\nadds the increase through `wrap_at`. When `wrap_at` is not set, it adds\nthe new value.",
"type": "string"
}
]
},
"DeriveTransformConfig": {
"description": "Writes a value onto each message that needs the previous message of the\nsame key. The value can be a rate of change, a delta, a running total or a\ncounter that wraps.\n\nYou can give many derivations. The transform writes each one to its own\n`as` field. The first message for each key has no previous message. For\nthat message, `rate` and `delta` write `null`.",
"properties": {
"derive": {
"description": "The values to calculate. Give one or more, each with a different\n`as`.",
"items": {
"$ref": "#/$defs/Derivation"
},
"type": "array"
},
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out for\none series.",
"items": {
"type": "string"
},
"type": "array"
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have a derived field or a\ngroup field. The default is `error`."
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"time": {
"description": "The field that holds the time of each message, as an RFC 3339 string\nor as milliseconds since the epoch. Leave it out to use the arrival\ntime.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"derive"
],
"title": "derive",
"type": "object"
},
"DetectLearn": {
"description": "The readings that a `detect` baseline learns from, for the methods that\ncontinue to learn.",
"oneOf": [
{
"const": "all",
"description": "All readings, with anomalies included. A change that continues becomes\nthe new normal at the speed of the baseline.",
"type": "string"
},
{
"const": "normal_only",
"description": "Only the readings that are not anomalies. An anomaly cannot change the\nbaseline. Use it with `readapt_after_seconds`. Without it, the\ntransform flags a real change of level permanently.",
"type": "string"
}
]
},
"DetectMethod": {
"description": "How the transform finds an anomaly.\n\nThe window methods (`zscore`, `mad`) compare a value with the values before\nit. The window does not include the value, so a spike does not change its\nown baseline. The chart methods (`cusum` without `target`, `ewma_chart`,\n`western_electric`) fix their baseline at the end of the warm-up.",
"oneOf": [
{
"description": "An anomaly is more than `threshold` standard deviations from the mean\nof the last `size` values.",
"properties": {
"size": {
"description": "The number of earlier values in the baseline.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"threshold": {
"description": "The number of standard deviations that makes an anomaly. The\ndefault is `3`.",
"format": "double",
"type": [
"number",
"null"
]
},
"type": {
"const": "zscore",
"type": "string"
}
},
"required": [
"type",
"size"
],
"type": "object"
},
{
"description": "An anomaly is more than `threshold` scaled median absolute deviations\nfrom the median of the last `size` values. Use it when the baseline\ncontains outliers.",
"properties": {
"size": {
"description": "The number of earlier values in the baseline.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"threshold": {
"description": "The number of scaled MADs that makes an anomaly. The default is\n`3.5`.",
"format": "double",
"type": [
"number",
"null"
]
},
"type": {
"const": "mad",
"type": "string"
}
},
"required": [
"type",
"size"
],
"type": "object"
},
{
"description": "A two-sided CUSUM. It adds the drift above `target` and the drift below\n`target` in two sums. An anomaly is a sum that is more than\n`threshold`. Then that sum goes back to zero. Use it to find a small\nshift that continues.",
"properties": {
"drift": {
"description": "The change per message that does not count as drift, in the units\nof the field.",
"format": "double",
"type": "number"
},
"target": {
"description": "The expected value of the series. Leave it out to use the mean of\nthe warm-up.",
"format": "double",
"type": [
"number",
"null"
]
},
"threshold": {
"description": "The sum of drift that makes an anomaly.",
"format": "double",
"type": "number"
},
"type": {
"const": "cusum",
"type": "string"
}
},
"required": [
"type",
"drift",
"threshold"
],
"type": "object"
},
{
"description": "An EWMA control chart. An anomaly is a smoothed value outside a band of\n`threshold` standard deviations around the mean of the warm-up. It\nfinds small shifts and ignores single points.",
"properties": {
"alpha": {
"description": "The weight of the newest value, from 0 to 1. The default is `0.2`.",
"format": "double",
"type": [
"number",
"null"
]
},
"threshold": {
"description": "The width of the band, in standard deviations. The default is `3`.",
"format": "double",
"type": [
"number",
"null"
]
},
"type": {
"const": "ewma_chart",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"description": "The Western Electric rules, against the mean and deviation of the\nwarm-up:\n\n- one point more than 3σ from the mean,\n- two of three points more than 2σ from the mean on one side,\n- four of five points more than 1σ from the mean on one side,\n- eight points in a sequence on one side.\n\nThe transform writes the rule that matched beside the flag.",
"properties": {
"type": {
"const": "western_electric",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"description": "An anomaly is `size` identical values in a sequence. Use it to find a\nstuck instrument.",
"properties": {
"size": {
"description": "The number of identical values in a sequence that makes an anomaly.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"type": {
"const": "flatline",
"type": "string"
}
},
"required": [
"type",
"size"
],
"type": "object"
},
{
"description": "An anomaly is more than `threshold` deviations from a baseline that\ncontinues to learn. The baseline is an exponentially weighted mean and\nspread, each with its own time constant. Use it for a series that\ndrifts slowly and arrives at irregular times. It reads the `time` field\nof the transform.",
"properties": {
"mean_tau_seconds": {
"description": "The time constant of the mean, in seconds. A smaller value makes the\nmean follow the series more quickly.",
"format": "double",
"type": "number"
},
"min_spread": {
"description": "The minimum deviation, in the units of the field. Without it, a\nsignal that was very quiet flags its first small change. The default\nis `0`.",
"format": "double",
"type": [
"number",
"null"
]
},
"spread_tau_seconds": {
"description": "The time constant of the spread, in seconds. Make it longer than\n`mean_tau_seconds`, so that a short period of noise does not make\nthe band wider immediately.",
"format": "double",
"type": "number"
},
"threshold": {
"description": "The number of deviations that makes an anomaly. The default is `3`.",
"format": "double",
"type": [
"number",
"null"
]
},
"type": {
"const": "ewma",
"type": "string"
}
},
"required": [
"type",
"mean_tau_seconds",
"spread_tau_seconds"
],
"type": "object"
}
]
},
"DetectMode": {
"description": "Which messages the `detect` transform sends.",
"oneOf": [
{
"const": "annotate",
"description": "Every message passes, with the flag and the score.",
"type": "string"
},
{
"const": "only_anomalies",
"description": "Only the anomalies pass, with the flag and the score.",
"type": "string"
}
]
},
"DetectTransformConfig": {
"description": "Flags anomalies in a numeric field against its own series. `method` selects\nhow the transform finds an anomaly.\n\nThe transform writes a boolean under `as`. The default is `anomaly`. It also\nwrites a score under `<as>_score`. The score is the distance from normal, in\nthe units of the method. A `filter` after the transform can use a stricter\nlimit on the score.\n\nThe warm-up is `min_samples` messages for each key. The transform flags\nnothing during the warm-up.",
"properties": {
"as": {
"description": "The field to write the flag to. The score goes to `<as>_score`. The\ndefault is `anomaly`.",
"type": [
"string",
"null"
]
},
"field": {
"description": "The numeric field to examine.",
"type": "string",
"x-message-field": true
},
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out for\none series.",
"items": {
"type": "string"
},
"type": "array"
},
"learn": {
"$ref": "#/$defs/DetectLearn",
"description": "For `zscore`, `mad` and `ewma`: the readings that the baseline learns\nfrom. The default is `all`."
},
"method": {
"$ref": "#/$defs/DetectMethod",
"description": "How the transform finds an anomaly."
},
"min_samples": {
"description": "The number of messages for each key before the transform flags\nanything. The default is the `size` of the method, or 30 for a method\nwith no `size`.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"mode": {
"$ref": "#/$defs/DetectMode",
"description": "Which messages to send. The default is `annotate`."
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have the field or a group\nfield. The default is `error`."
},
"readapt_after_seconds": {
"description": "With `learn: normal_only`: when readings are anomalies for this number\nof seconds without a break, the baseline learns from them. A change\nthat continues then becomes the new normal.",
"format": "double",
"type": [
"number",
"null"
]
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"time": {
"description": "The field that holds the time of each message, as an RFC 3339 string\nor as milliseconds since the epoch. The `ewma` method and\n`readapt_after_seconds` use it. Leave it out to use the arrival time.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"with_baseline": {
"description": "Also write the baseline of the reading, in the units of the field. The\nnormal value goes to `<as>_expected`. The permitted distance from it\ngoes to `<as>_band`. The value is `null` when the method has no\nbaseline.",
"type": "boolean"
}
},
"required": [
"field",
"method"
],
"title": "detect",
"type": "object"
},
"FeatureKind": {
"description": "A number that describes a window of readings.",
"oneOf": [
{
"const": "mean",
"description": "The arithmetic mean.",
"type": "string"
},
{
"const": "std",
"description": "The population standard deviation.",
"type": "string"
},
{
"const": "min",
"description": "The smallest value.",
"type": "string"
},
{
"const": "max",
"description": "The largest value.",
"type": "string"
},
{
"const": "range",
"description": "The largest value minus the smallest value.",
"type": "string"
},
{
"const": "slope",
"description": "The least-squares slope. It is per second with the `time` field, and\nper message without it.",
"type": "string"
},
{
"const": "skew",
"description": "The skewness: the direction of the longer tail.",
"type": "string"
},
{
"const": "kurtosis",
"description": "The excess kurtosis: the weight of the tails.",
"type": "string"
},
{
"const": "rms",
"description": "The root mean square.",
"type": "string"
},
{
"const": "crest_factor",
"description": "The peak magnitude divided by the RMS.",
"type": "string"
},
{
"const": "zero_crossings",
"description": "The number of times that the signal crosses zero.",
"type": "string"
},
{
"const": "n_peaks",
"description": "The number of local maxima.",
"type": "string"
},
{
"const": "autocorr1",
"description": "The autocorrelation at lag one.",
"type": "string"
},
{
"const": "dominant_frequency",
"description": "The strongest frequency above DC, in hertz. It needs a sample rate from\nthe `time` field or from `sample_rate_hz`.",
"type": "string"
},
{
"const": "count",
"description": "The number of readings in the window.",
"type": "string"
},
{
"const": "duration",
"description": "The time from the first reading to the last reading, in seconds, from\nthe `time` field.",
"type": "string"
}
]
},
"FeaturesTransformConfig": {
"description": "Changes a batch of readings into one message of features for each group.\nUse it to send a small set of numbers to a model, in place of the raw\nreadings. Put a `buffer` on the input. Without it, each batch has only one\nreading.\n\nThe transform writes:\n\n- each feature in `include` under its own name, for example `mean`, `rms`\n or `crest_factor`,\n- each entry in `bands` under its `as`,\n- the `group_by` fields under their leaf names, as `reduce` does.\n\nA feature with no value for the window is `null`. For example, a slope of\none point is `null`. The spectral features (`dominant_frequency`, `bands`)\nneed a sample rate. kayak uses `sample_rate_hz`, or calculates the rate\nfrom the `time` field. Without one of the two, the pipeline does not build.\nThis transform keeps no state and needs no state bucket.",
"properties": {
"bands": {
"description": "The frequency bands to calculate the power of. The transform writes\neach one under its `as`. Give `include`, `bands` or both.",
"items": {
"$ref": "#/$defs/Band"
},
"type": "array"
},
"field": {
"description": "The numeric field to read.",
"type": "string",
"x-message-field": true
},
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out to use\nthe full batch as one window.",
"items": {
"type": "string"
},
"type": "array"
},
"include": {
"description": "The features to calculate. The transform writes each one under its own\nname.",
"items": {
"$ref": "#/$defs/FeatureKind"
},
"type": "array"
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a reading that does not have the field or a group\nfield. The default is `error`."
},
"sample_rate_hz": {
"description": "The sample rate of the readings in hertz, for the spectral features.\nIt must be more than zero. kayak uses it in place of the rate from\n`time`. Use it when the source has timestamps with low resolution.",
"format": "double",
"type": [
"number",
"null"
]
},
"time": {
"description": "The field that holds the time of each reading, as an RFC 3339 string or\nas milliseconds since the epoch. `slope` and `duration` use it for\nseconds. The spectral features use it for the sample rate.",
"type": [
"string",
"null"
],
"x-message-field": true
}
},
"required": [
"field"
],
"title": "features",
"type": "object"
},
"FilterTransformConfig": {
"description": "Keeps the messages that match all `conditions` and drops the other\nmessages. With `invert`, it drops the messages that match and keeps the\nother messages.\n\nThe transform drops a batch that has no messages left. A message that does\nnot have the field of a condition does not match that condition. A message\nwith a field of the wrong type also does not match. Thus `invert` keeps\nthese messages.",
"properties": {
"conditions": {
"description": "The conditions that a message must match. At least one condition is\nrequired.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"invert": {
"description": "Drop the messages that match, and keep the other messages. The default\nis false.",
"type": "boolean"
}
},
"required": [
"conditions"
],
"title": "filter",
"type": "object"
},
"HttpAuthConfig": {
"description": "A credential in a header. The `http` input checks it on each post. The\n`http` output, the `http` transform and the `http_poll` input send it on\neach request.\n\nThis credential is for one pipeline only. It is not related to the user\naccounts of the server.\n\nThe sender sends the same token on each request. Thus the token is only as\nsecure as the connection. kayak serves plain HTTP. Put TLS in front of\nkayak, or other systems on the network path can read the token.",
"oneOf": [
{
"description": "A token in the standard `Authorization` header, as\n`Authorization: Bearer TOKEN`. Use this variant if the other system can\nuse that header.",
"properties": {
"token": {
"$ref": "#/$defs/Secret",
"description": "The token. Use a `${NAME}` reference, so that the config file keeps\nonly the name and the secret store keeps the value."
},
"type": {
"const": "bearer",
"type": "string"
}
},
"required": [
"type",
"token"
],
"type": "object"
},
{
"description": "A fixed value in a header that you name. Use this variant for a system\nthat cannot use the `Authorization` header.",
"properties": {
"name": {
"description": "The name of the header. The `http` input compares the name without\ncase. On an `http` input, the name must not be a header that an\n`envelope` copies into the messages.",
"type": "string"
},
"type": {
"const": "header",
"type": "string"
},
"value": {
"$ref": "#/$defs/Secret",
"description": "The exact value of the header. Use a `${NAME}` reference."
}
},
"required": [
"type",
"name",
"value"
],
"type": "object"
}
]
},
"HttpBodyKind": {
"description": "The body of one request from an `http` output or an `http` transform.\n\nSelect the value that the API at the endpoint expects. Use `batch` for an\nendpoint that takes an array. Use `message` for a webhook that takes one\nevent for each call.",
"oneOf": [
{
"const": "batch",
"description": "The full batch as one JSON array, in one request. The default.",
"type": "string"
},
{
"const": "message",
"description": "One request for each message. The body is the message. The requests go\nin sequence. The first failure fails the batch, and kayak does not send\nthe messages after it.",
"type": "string"
}
]
},
"HttpResponseKind": {
"description": "What an `http` transform does with the reply.",
"oneOf": [
{
"const": "replace",
"description": "The reply becomes the new batch. With `body: batch`, the reply must be a\nJSON array of messages. With `body: message`, the reply is a message or\nan array of messages. The default.",
"type": "string"
},
{
"const": "merge",
"description": "The transform writes the reply onto the message that caused it, under\n`as`. With `body: batch`, an array reply with one entry for each message\ngoes to the messages in sequence. Any other reply goes onto each\nmessage. The messages keep all their fields.",
"type": "string"
}
]
},
"HttpTransformConfig": {
"description": "Sends the batch to an http endpoint and continues with the reply. Use it\nto call a model or another service.\n\n`body` sets the content of one request: the full batch as a JSON array, or\none message. `wrap` puts the body under a key, for example\n`{\"instances\": …}`. `response` sets what the transform does with the reply.\n`replace` makes the reply the new batch. `merge` writes the reply onto the\nmessage under `as`. `unwrap` reads the reply from under a key first.\n\nA status other than 2xx fails the batch. The error contains the reply of the\nendpoint. A network failure, a 5xx or a 429 is tried again `retries` times\nwith backoff.\n\nFor a model, put a `buffer` and a `features` transform before this\ntransform. Use `response: merge` to keep the identifiers of the message.",
"properties": {
"as": {
"description": "The field to write the reply under. Applies to `response: merge`.",
"type": [
"string",
"null"
]
},
"auth": {
"anyOf": [
{
"$ref": "#/$defs/HttpAuthConfig"
},
{
"type": "null"
}
],
"description": "The credential that the transform sends. If you do not set it, the\ntransform sends no credential."
},
"body": {
"anyOf": [
{
"$ref": "#/$defs/HttpBodyKind"
},
{
"type": "null"
}
],
"description": "The content of one request. The default is `batch`."
},
"response": {
"anyOf": [
{
"$ref": "#/$defs/HttpResponseKind"
},
{
"type": "null"
}
],
"description": "What to do with the reply. The default is `replace`."
},
"retries": {
"description": "The number of times to send a request again before the batch fails.\nApplies to a network failure, a 5xx and a 429. The default is 0. Each\ntry waits longer than the previous try, and the pipeline waits too.",
"format": "uint32",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"timeout_seconds": {
"description": "The maximum time for one request, in s. The default is 30.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"unwrap": {
"description": "A key to read the reply from, for an API that replies with\n`{\"predictions\": …}`.",
"type": [
"string",
"null"
]
},
"url": {
"description": "The endpoint to send to.",
"type": "string"
},
"verb": {
"$ref": "#/$defs/HttpVerb",
"description": "The http method. `GET` and `DELETE` fail to build, because a request\nwith no body cannot send the messages."
},
"wrap": {
"description": "A key to put the body under, for an API that expects `{\"key\": …}`.",
"type": [
"string",
"null"
]
}
},
"required": [
"url",
"verb"
],
"title": "http",
"type": "object"
},
"HttpVerb": {
"description": "The http method of a request.",
"enum": [
"GET",
"POST",
"PUT",
"PATCH",
"DELETE"
],
"type": "string"
},
"KeepPolicy": {
"description": "Which fields of the input message a `map` keeps.",
"oneOf": [
{
"const": "all",
"description": "Keep all fields, and apply the mappings to them. This is the default.",
"type": "string"
},
{
"const": "mapped",
"description": "Keep only the fields that the mappings wrote. Use it to prepare a\nmessage for an output with a fixed shape, for example a `postgres`\ntable. It also removes intermediate fields. You cannot use `drop` with\n`mapped`.",
"type": "string"
}
]
},
"Literal": {
"description": "A literal value. A `constant` writes it, and a `default` writes it in place\nof a missing field. The `type` field selects the type of the value.",
"oneOf": [
{
"description": "A string.",
"properties": {
"type": {
"const": "text",
"type": "string"
},
"value": {
"description": "The text.",
"type": "string"
}
},
"required": [
"type",
"value"
],
"type": "object"
},
{
"description": "A number.",
"properties": {
"type": {
"const": "number",
"type": "string"
},
"value": {
"description": "The number.",
"format": "double",
"type": "number"
}
},
"required": [
"type",
"value"
],
"type": "object"
},
{
"description": "True or false.",
"properties": {
"type": {
"const": "boolean",
"type": "string"
},
"value": {
"description": "The boolean value.",
"type": "boolean"
}
},
"required": [
"type",
"value"
],
"type": "object"
},
{
"description": "JSON `null`. The field is present with the value `null`.",
"properties": {
"type": {
"const": "null",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
}
]
},
"MapMissingPolicy": {
"description": "What `map` does with a message that does not have a field that a mapping\nreads.\n\nNo value drops the message. To drop a message, use `filter`.",
"oneOf": [
{
"const": "error",
"description": "Fail the batch. This is the default. To accept a missing field, use\n`omit`, or give the mapping a `default`.",
"type": "string"
},
{
"const": "omit",
"description": "Do not write the target field.",
"type": "string"
},
{
"const": "null",
"description": "Write the target field as `null`.",
"type": "string"
}
]
},
"MapTransformConfig": {
"description": "Changes the shape of every message. A mapping can rename, move, cast or\nremove a field, or write a constant. The transform applies the mappings in\norder.\n\nEach entry in `mappings` reads fields from the message and writes one\nfield. A mapping can read the fields that earlier mappings wrote. Use this\nfor intermediate values. With `keep: all`, a `drop` can remove them again.\n\nReads and writes use dotted paths. For example, an `as` of `sensor.id`\nwrites the value inside a `sensor` object. If the object does not exist,\nthe transform makes it.\n\nBy default, the message passes through with the mappings applied to it.\n`keep` can change this. One message goes in and one message comes out. To\ndrop a message, use `filter`. To make many messages, use `splitter`.",
"properties": {
"keep": {
"$ref": "#/$defs/KeepPolicy",
"description": "Which fields of the input message the output keeps. The default is\n`all`."
},
"mappings": {
"description": "The fields to write, in order. Give one mapping or more. Two mappings\nmust not write the same field.",
"items": {
"$ref": "#/$defs/Mapping"
},
"type": "array"
},
"on_missing": {
"$ref": "#/$defs/MapMissingPolicy",
"description": "What to do with a message that does not have a field that a mapping\nreads. The default is `error`. A `default` on the mapping applies\nfirst. Use a `default` when you expect one field to be absent."
}
},
"required": [
"mappings"
],
"title": "map",
"type": "object"
},
"Mapping": {
"description": "One field that the transform writes onto the message, and the source of\nits value. The `type` field selects the mapping.",
"oneOf": [
{
"description": "Reads a value from one field and writes it to another field. Use it to\nrename a field, or to move a field out of a nested object, for example\nfrom `_meta.subject` to `subject`.",
"properties": {
"as": {
"description": "The field to write. The default is the last segment of `from`.",
"type": [
"string",
"null"
]
},
"default": {
"anyOf": [
{
"$ref": "#/$defs/Literal"
},
{
"type": "null"
}
],
"description": "The value to write when the message does not have `from`. With it,\n`on_missing` does not apply."
},
"from": {
"description": "The field to read, as a dotted path.",
"type": "string"
},
"type": {
"const": "copy",
"type": "string"
}
},
"required": [
"type",
"from"
],
"type": "object"
},
{
"description": "Writes a fixed value, for example the name of the site.",
"properties": {
"as": {
"description": "The field to write the value to.",
"type": "string"
},
"type": {
"const": "constant",
"type": "string"
},
"value": {
"$ref": "#/$defs/Literal",
"description": "The value to write."
}
},
"required": [
"type",
"value",
"as"
],
"type": "object"
},
{
"description": "Writes the value of the first field in a list that the message has.\nUse it when two sources use different names for the same field.",
"properties": {
"as": {
"description": "The field to write the first value to.",
"type": "string"
},
"default": {
"anyOf": [
{
"$ref": "#/$defs/Literal"
},
{
"type": "null"
}
],
"description": "The value to write when the message has none of the fields."
},
"from": {
"description": "The fields to try, in order. Give two fields or more.",
"items": {
"type": "string"
},
"type": "array"
},
"type": {
"const": "coalesce",
"type": "string"
}
},
"required": [
"type",
"from",
"as"
],
"type": "object"
},
{
"description": "Converts a value from one JSON type to another. For example, it\nconverts the string `\"12.5\"` to the number `12.5`, or epoch seconds to\na timestamp. It can also parse a string that contains JSON.\n\nThis is the only place in kayak that converts a value. The column\nmapping of the database outputs checks a value and does not convert it.",
"properties": {
"as": {
"description": "The field to write. The default is the last segment of `from`. For\nexample, `{\"from\": \"value\", \"to\": \"float\"}` converts `value` in place.",
"type": [
"string",
"null"
]
},
"default": {
"anyOf": [
{
"$ref": "#/$defs/Literal"
},
{
"type": "null"
}
],
"description": "The value to write when the message does not have `from`. A value\nthat is present and does not convert is always an error."
},
"from": {
"description": "The field to read.",
"type": "string"
},
"to": {
"$ref": "#/$defs/CastType",
"description": "The type to convert the value to."
},
"type": {
"const": "cast",
"type": "string"
}
},
"required": [
"type",
"from",
"to"
],
"type": "object"
},
{
"description": "Joins fields and literal text into one string. For example, use it to\nwrite a `site/machine` key for `group_by`.",
"properties": {
"as": {
"description": "The field to write the string to.",
"type": "string"
},
"parts": {
"description": "The parts, in order. Give one part or more.",
"items": {
"$ref": "#/$defs/ConcatPart"
},
"type": "array"
},
"type": {
"const": "concat",
"type": "string"
}
},
"required": [
"type",
"parts",
"as"
],
"type": "object"
},
{
"description": "One arithmetic operation on two numbers. Each number is a field or a\nliteral.\n\nFor a calculation with more steps, use more mappings with intermediate\nfields. For example, `(f - 32) / 1.8` is two mappings. For a long\ncalculation, use the `script` transform.",
"properties": {
"as": {
"description": "The field to write the result to.",
"type": "string"
},
"left": {
"$ref": "#/$defs/Operand",
"description": "The left operand."
},
"on_zero": {
"$ref": "#/$defs/OnZero",
"description": "For `divide`: what to do when the right field holds zero. The\ndefault is `error`, which fails the batch."
},
"operator": {
"$ref": "#/$defs/ArithmeticOperator",
"description": "The operation to do."
},
"right": {
"$ref": "#/$defs/Operand",
"description": "The right operand."
},
"type": {
"const": "arithmetic",
"type": "string"
}
},
"required": [
"type",
"left",
"operator",
"right",
"as"
],
"type": "object"
},
{
"description": "Writes the start of the calendar period that a time is in, for example\nthe hour, the day or the shift.\n\nUse the field in `group_by`. A stateful transform that groups by it\nkeeps one series for each period. The idle timeout of the state bucket\nremoves the old periods.\n\nThe periods use the local clock of `timezone`. Thus, a shift that\nstarts at 06:00 starts at 06:00 in summer and in winter. On the night\nthat the clock changes, the shift is 7 or 9 hours.\n\nThe periods start at local midnight on 1 January 1970, plus\n`offset_seconds`. For example, `every_seconds: 28800` with\n`offset_seconds: 21600` gives 06:00, 14:00 and 22:00. A week starts on a\nThursday. For a week that starts on a Monday, add an offset of four days.",
"properties": {
"as": {
"description": "The field to write the start time to.",
"type": "string"
},
"every_seconds": {
"description": "The length of a period, in seconds.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"format": {
"anyOf": [
{
"$ref": "#/$defs/TimeFormat"
},
{
"type": "null"
}
],
"description": "The format of the start time. The default is `rfc3339`."
},
"from": {
"description": "The field that holds the time, as an RFC 3339 string or as\nmilliseconds since the epoch.",
"type": "string"
},
"offset_seconds": {
"description": "The offset of the start of the periods, in seconds. It must be\nless than `every_seconds`.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"timezone": {
"description": "The IANA time zone of the periods, for example `Europe/Stockholm`.\nThe default is UTC.",
"type": [
"string",
"null"
]
},
"type": {
"const": "time_bucket",
"type": "string"
}
},
"required": [
"type",
"from",
"every_seconds",
"as"
],
"type": "object"
},
{
"description": "Removes fields from the message. Use it to remove metadata fields\nbefore the output. A field that is not there is not an error.\n`on_missing` does not apply.",
"properties": {
"from": {
"description": "The fields to remove. Give one field or more.",
"items": {
"type": "string"
},
"type": "array"
},
"type": {
"const": "drop",
"type": "string"
}
},
"required": [
"type",
"from"
],
"type": "object"
}
]
},
"MissingFieldPolicy": {
"description": "What to do with a message that does not have a field that is aggregated or\ngrouped by. A field with the value `null` is missing.",
"oneOf": [
{
"const": "error",
"description": "Fail the batch. The default. A sum of only some of the messages gives a\nwrong result that is not visible downstream.",
"type": "string"
},
{
"const": "skip",
"description": "Do not use that message in that aggregation. An aggregation with no\nvalues gives `null`, or `0` for the counts.",
"type": "string"
}
]
},
"NumericFilterOperatorKind": {
"description": "How a `numeric` condition compares a number to the value in the config.",
"enum": [
"greater_than",
"less_than",
"equal_to",
"not_equal_to"
],
"type": "string"
},
"OnZero": {
"description": "What a `divide` does when the right field holds zero. For example, this\noccurs with a ratio over an empty period.",
"oneOf": [
{
"description": "Fail the batch. The error names the division. This is the default.",
"properties": {
"type": {
"const": "error",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"description": "Write `null` as the result.",
"properties": {
"type": {
"const": "null",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"description": "Write a number as the result.",
"properties": {
"type": {
"const": "value",
"type": "string"
},
"value": {
"description": "The number to write.",
"format": "double",
"type": "number"
}
},
"required": [
"type",
"value"
],
"type": "object"
}
]
},
"Operand": {
"description": "One side of an [`Mapping::Arithmetic`]: a field to read, or a fixed number.",
"oneOf": [
{
"description": "A number from the message.",
"properties": {
"field": {
"description": "The field to read. It must hold a number.",
"type": "string"
},
"type": {
"const": "field",
"type": "string"
}
},
"required": [
"type",
"field"
],
"type": "object"
},
{
"description": "A number in the config.",
"properties": {
"type": {
"const": "value",
"type": "string"
},
"value": {
"description": "The number.",
"format": "double",
"type": "number"
}
},
"required": [
"type",
"value"
],
"type": "object"
}
]
},
"PipelineState": {
"description": "The state bucket of a pipeline, and the field that gives the key of each\nmessage.\n\nEach pipeline sets its own key, because the same value can have different\nfield names in two streams. For example, one stream has\n`_meta.machine_id` and another stream has `machine_id`.\n\nMake sure that all pipelines that share a bucket use keys with the same\nvalues. kayak does not check this.",
"properties": {
"bucket": {
"description": "The name of the bucket that this pipeline reads and writes. Declare the\nbucket under `state` at the top of the config. If the bucket is not\ndeclared, the pipeline does not build.",
"type": "string"
},
"key": {
"description": "The field that gives the key, as a dotted path. For example,\n`_meta.machine_id`.\n\nLeave it out for one value for the full bucket. Use that only for an\nitem that has one value. For a value for each device, set `key`.",
"type": [
"string",
"null"
]
}
},
"required": [
"bucket"
],
"title": "pipeline state",
"type": "object"
},
"PivotTransformConfig": {
"description": "Changes a stream with one reading per message into rows. The transform\nkeeps the latest value of each name in `names` for each key. It writes all\nof these values onto every message.\n\nFor example, the input is `{\"sensor\": \"state\", \"value\": \"RUNNING\"}` and\nthen `{\"sensor\": \"fault\", \"value\": \"NONE\"}`. The output row is\n`{\"state\": \"RUNNING\", \"fault\": \"NONE\", ...}`.\n\nWhen the `name` field of a message holds one of `names`, the transform\nfirst records that reading. Then it writes every value that it keeps for\nthe key onto the message. It writes them at the top level, or under\n`into`. A name with no reading yet for the key is not written. One message\ngoes in and one message comes out.\n\n`names` is required because it limits the state. A message with a name that\nis not in `names` adds nothing, but it gets the row.",
"properties": {
"group_by": {
"description": "The fields that identify a row, as in `reduce`. Leave it out for one\nrow.",
"items": {
"type": "string"
},
"type": "array"
},
"into": {
"description": "An object field to write the values under. Leave it out to write them\nat the top level.",
"type": [
"string",
"null"
]
},
"name": {
"description": "The field that holds the name of the reading. kayak compares its value\nwith `names`.",
"type": "string",
"x-message-field": true
},
"names": {
"description": "The names to keep and write. Each name becomes a field. Give one name\nor more.",
"items": {
"type": "string"
},
"type": "array"
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have a group field, or that\nhas a name from `names` and no `value`. The default is `error`."
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"value": {
"description": "The field that holds the reading. The value can be any JSON value, for\nexample a string or a number.",
"type": "string",
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"name",
"value",
"names"
],
"title": "pivot",
"type": "object"
},
"RecallMissingPolicy": {
"description": "What `recall` does when the bucket has no value for the key of a message.\n\nThe default is `skip`, because the bucket is empty when the pipeline\nstarts. With `error`, the batches fail until the bucket has values.",
"oneOf": [
{
"const": "skip",
"description": "Drop the message. The default. Without the recalled values, a reducer\nafter this transform puts all these messages into one wrong group.",
"type": "string"
},
{
"const": "null",
"description": "Send the message on with the missing names as `null`.",
"type": "string"
},
{
"const": "error",
"description": "Fail the batch. Use it only when another component always fills the\nbucket first.",
"type": "string"
}
]
},
"RecallTransformConfig": {
"description": "Writes values from the state bucket of the pipeline onto each message,\nwith the names from `remember`.\n\nUse it to add a slow fact to a fast stream, for example the current recipe\nof a machine. The values become top-level fields, so a `reducer` after it\ncan group by them.\n\nThe pipeline must have a `state`, or the transform fails to build.",
"properties": {
"on_missing": {
"$ref": "#/$defs/RecallMissingPolicy",
"description": "What to do with a message when the bucket has no value for its key.\nThe default is `skip`."
},
"recall": {
"description": "The names to read from the bucket, as `remember` wrote them. The\ntransform writes each value onto the message with the same name.",
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"recall"
],
"title": "recall",
"type": "object"
},
"ReduceFnKind": {
"description": "How a reducer combines the values of one field into one result.",
"oneOf": [
{
"const": "sum",
"description": "The total. Numbers only.",
"type": "string"
},
{
"const": "avg",
"description": "The arithmetic mean. Numbers only.",
"type": "string"
},
{
"const": "min",
"description": "The smallest value. Numbers compare as numbers. Strings compare in\nalphabetical sequence, so `min` of ISO timestamps is the earliest time.",
"type": "string"
},
{
"const": "max",
"description": "The largest value. Values compare as for `min`.",
"type": "string"
},
{
"const": "count",
"description": "The number of messages. This function does not need a `field`. With a\n`field`, it counts the messages that have the field.",
"type": "string"
},
{
"const": "count_distinct",
"description": "The number of different values. The function compares the values as\nJSON.",
"type": "string"
},
{
"const": "first",
"description": "The value from the first message of the group, of any type.",
"type": "string"
},
{
"const": "last",
"description": "The value from the last message of the group.",
"type": "string"
},
{
"const": "collect",
"description": "All values as an array, in the sequence of arrival.",
"type": "string"
},
{
"const": "median",
"description": "The middle value, or the mean of the two middle values. Numbers only.",
"type": "string"
},
{
"const": "stddev",
"description": "The population standard deviation. Numbers only.",
"type": "string"
},
{
"const": "slope",
"description": "The rate of change of the field per second, from a least-squares line\nagainst the time of each message. Numbers only. The reducer must have a\n`time` setting, or it fails to build.",
"type": "string"
}
]
},
"ReduceTransformConfig": {
"description": "Reduces a batch to one message for each group. Put a buffer before it, or\nit gets only one message at a time.\n\nWithout `group_by`, the full batch is one group and the reducer sends one\nmessage. With `group_by`, it sends one message for each different\ncombination of those fields. The messages are in the sequence in which the\ngroups first occur. Each message contains the `group_by` fields and the\nresults of the aggregations.\n\nEach aggregation has a `function`, the `field` to use and the name `as` for\nthe result, for example\n`{\"function\": \"avg\", \"field\": \"value\", \"as\": \"mean\"}`.",
"properties": {
"aggregations": {
"description": "The values to calculate. At least one aggregation is required. Each one\nmust have a different `as`.",
"items": {
"$ref": "#/$defs/Aggregation"
},
"type": "array"
},
"group_by": {
"description": "The fields whose combination defines a group. If you do not set it, the\nfull batch is one group.",
"items": {
"type": "string"
},
"type": "array"
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have one of the fields. The\ndefault is `error`."
},
"time": {
"description": "The field that contains the time of each message, as an RFC 3339 string\nor as ms since the epoch. `slope` requires it. A message without this\nfield fails the batch. If you do not set it, the time of each message is\nits arrival time.",
"type": [
"string",
"null"
],
"x-message-field": true
}
},
"required": [
"aggregations"
],
"title": "reducer",
"type": "object"
},
"RememberTransformConfig": {
"description": "Writes values from the messages that match into the state bucket of the\npipeline. The key is the field that `state.key` of the pipeline names.\n\nThe transform sends each message on without changes. It does not filter.\n\nThe pipeline must have a `state`, or the transform fails to build.",
"properties": {
"remember": {
"description": "The values to take from a message that matches. At least one entry is\nrequired. Each entry must have a different `as`.",
"items": {
"$ref": "#/$defs/Remembered"
},
"type": "array"
},
"when": {
"description": "The conditions that a message must match to be remembered. All of them\nmust match. If you do not set it, the transform remembers values from\neach message. Set it when the stream contains more than one type of\nmessage.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"remember"
],
"title": "remember",
"type": "object"
},
"Remembered": {
"description": "One value to write into the state bucket of the pipeline, and its name\nthere.",
"properties": {
"as": {
"description": "The name for the value in the bucket. `recall` reads the value by this\nname. Each entry must have a different name.",
"type": "string"
},
"field": {
"description": "The field to take the value from.",
"type": "string"
}
},
"required": [
"field",
"as"
],
"type": "object"
},
"ResampleMethod": {
"description": "How the transform calculates the value at a grid point from the readings in\nits interval, and what it does with an empty interval.",
"oneOf": [
{
"const": "last",
"description": "The last reading in the interval. An empty interval sends nothing.",
"type": "string"
},
{
"const": "mean",
"description": "The mean of the readings in the interval. An empty interval sends\nnothing.",
"type": "string"
},
{
"const": "linear",
"description": "The value on a straight line between the last reading before the grid\npoint and the first reading after it. The transform sends the grid\npoint when the reading after it arrives. The same line fills the empty\nintervals between the two readings.",
"type": "string"
},
{
"const": "forward_fill",
"description": "The last reading, carried forward. An empty interval repeats the last\nvalue for a maximum of `max_gap_seconds`, and then stops. This is the\nonly method that sends values from a quiet series. Use it to change a\nsignal that reports on change into a regular signal.",
"type": "string"
}
]
},
"ResampleTransformConfig": {
"description": "Puts a series onto a regular grid. The transform sends one message for each\nkey in each `interval_seconds`, at times that are multiples of the interval.\nThe rate of the readings has no effect on the grid.\n\nThe message that the transform sends contains:\n\n- the group fields, under their leaf names,\n- the grid time as an RFC 3339 string, under the name of the `time` field\n (or `time` when the transform uses the arrival time),\n- the value, under `as` (the leaf of `field` when you leave it out).\n\nThe transform sends a grid point when a later reading arrives. With\n`forward_fill` and the arrival time, it also sends a grid point when the\nclock passes it.",
"properties": {
"as": {
"description": "The field to write the value to. The default is the leaf of `field`.",
"type": [
"string",
"null"
]
},
"field": {
"description": "The numeric field to resample.",
"type": "string",
"x-message-field": true
},
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out for\none series.",
"items": {
"type": "string"
},
"type": "array"
},
"interval_seconds": {
"description": "The distance between two grid points, in seconds.",
"format": "double",
"type": "number"
},
"max_gap_seconds": {
"description": "For `forward_fill`: the maximum time to repeat a value into empty\nintervals, in seconds. Leave it out to repeat the value with no limit.",
"format": "double",
"type": [
"number",
"null"
]
},
"method": {
"$ref": "#/$defs/ResampleMethod",
"description": "How the transform calculates the value of an interval."
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have the field or a group\nfield. The default is `error`."
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"time": {
"description": "The field that holds the time of each message, as an RFC 3339 string\nor as milliseconds since the epoch. Leave it out to use the arrival\ntime. With the arrival time, the clock finds the empty intervals.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"field",
"interval_seconds",
"method"
],
"title": "resample",
"type": "object"
},
"RollingTransformConfig": {
"description": "Writes aggregations over the last messages of the series onto each\nmessage. The aggregations use the same `{function, field, as}` list and\nfunctions as `reduce`. One message goes in and one message comes out.\n\nThe window holds a maximum of `size` messages. `size` is always required.\nWith `seconds`, the window also drops messages that are older than that\ntime, from the `time` field.\n\nHere, `count` needs a `field`. It counts the messages in the window that\nhave the field. When the window is full, the count is `size`.",
"properties": {
"aggregations": {
"description": "The aggregations to calculate over the window. Give one or more, each\nwith a different `as`.",
"items": {
"$ref": "#/$defs/Aggregation"
},
"type": "array"
},
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out for\none series.",
"items": {
"type": "string"
},
"type": "array"
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have an aggregated field or a\ngroup field. The default is `error`."
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"seconds": {
"description": "Also drop the messages that are older than this number of seconds,\nfrom the `time` field.",
"format": "double",
"type": [
"number",
"null"
]
},
"size": {
"description": "The maximum number of messages in the window.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"time": {
"description": "The field that holds the time of each message, as an RFC 3339 string\nor as milliseconds since the epoch. Leave it out to use the arrival\ntime.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"aggregations",
"size"
],
"title": "rolling",
"type": "object"
},
"ScriptScope": {
"description": "Whether the script gets one message or the full batch.\n\n`message` is the default. The operation limit then applies to each message,\nand the batch keeps its structure.\n\nUse `batch` for work on the full batch. For example, remove duplicates in\nthe batch, or calculate a value that `reduce` has no function for.",
"oneOf": [
{
"const": "message",
"description": "The script runs one time for each message, with the message in `msg`.",
"type": "string"
},
{
"const": "batch",
"description": "The script runs one time for each batch, with the messages in `batch`\nas an array. When you emit an array, the transform sends a batch of\nthose messages.",
"type": "string"
}
]
},
"ScriptSource": {
"description": "The source of the script text: `inline` or `file`.\n\nUse `inline` to send a script through the HTTP API. In a YAML config, an\ninline script is a literal block. Use `file` to keep the script in its own\nfile, where an editor, a formatter and a test can use it. A save keeps the\n`file` reference in the config. It does not copy the script into the\nconfig.",
"oneOf": [
{
"description": "The script text, in the config. Use a YAML config for an inline script.\nYAML keeps the script as a literal block. JSON must escape each newline.",
"properties": {
"code": {
"description": "The rhai source.",
"type": "string",
"x-script": "rhai"
},
"type": {
"const": "inline",
"type": "string"
}
},
"required": [
"type",
"code"
],
"type": "object"
},
{
"description": "A path to a `.rhai` file, relative to the directory of the config file.\n\nkayak reads the file and its imports when it builds the pipeline. After\nyou change the file, do a revert to use the change. A server with no\nconfig file does not accept a `file` source. Inline scripts work on that\nserver, but their imports do not.",
"properties": {
"path": {
"description": "The path, relative to the directory of the config file. The path\nmust stay in that directory.",
"type": "string"
},
"type": {
"const": "file",
"type": "string"
}
},
"required": [
"type",
"path"
],
"type": "object"
}
]
},
"ScriptTransformConfig": {
"description": "Runs a [rhai](https://rhai.rs) script over each message, or over the full\nbatch, and sends the values that the script emits.\n\nThe script gets the message as `msg`, and sends values with `emit(value)`.\nCall `emit` zero times to drop the message, one time to replace it, or more\ntimes to split it. Use a script when `filter`, `map` and `splitter` are not\nsufficient.\n\nkayak **compiles the script when it builds the pipeline**. Thus, a syntax\nerror stops the pipeline from starting. An error that occurs only with a\nmessage fails that batch. For example, a missing field or a value that does\nnot convert fails the batch.\n\nEach run of the script has an **operation limit** (`max_operations`). When\nthe script reaches the limit, kayak stops it and fails the batch. Thus, a\nscript with an endless loop cannot block the pipeline.\n\nA script can **`import`** other rhai files. Give a literal path relative to\nthe directory of the config file. The path must stay in that directory. You\ncan leave out the `.rhai` extension. kayak reads the imports when it builds\nthe pipeline. Thus, a bad import stops the pipeline from starting, and a\nrunning script does not read the filesystem.",
"properties": {
"max_operations": {
"description": "The maximum number of rhai operations in one run of the script. At the\nlimit, kayak stops the script and fails the batch. The default is\n100000. Increase it for a script that walks a large array.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"scope": {
"$ref": "#/$defs/ScriptScope",
"description": "Whether the script gets one message at a time or the full batch. The\ndefault is `message`."
},
"source": {
"$ref": "#/$defs/ScriptSource",
"description": "The script: inline in the config, or in a file beside the config."
}
},
"required": [
"source"
],
"title": "script",
"type": "object"
},
"Secret": {
"description": "A text value that can refer to secrets with `${NAME}` references.\n\nThe value is an ordinary JSON string. When kayak builds the pipeline, it\nreplaces each `${NAME}` reference with the value from the secret store of\nthe server:\n\n```json\n{ \"type\": \"nats\", \"urls\": \"nats://app:${NATS_PASSWORD}@broker:4222\" }\n```\n\nThe config keeps only the reference, never the value. Thus you can commit\nthe file, and `GET /api/pipelines` does not show the secret.\n\nkayak uses a value with no `${...}` reference as it is.",
"type": "string"
},
"SmoothMethod": {
"description": "How the transform smooths a value against the values before it.",
"oneOf": [
{
"description": "An exponentially weighted moving average. It needs no window. Give\nexactly one of `alpha`, `half_life` or `tau_seconds`.\n\n`alpha` and `half_life` count messages. Use them when messages arrive\nat a steady rate. `tau_seconds` counts time. The weight of a value is\n`1 − e^(−Δt/τ)`, where Δt is the time since the previous value. Thus, a\nreading after a long gap has more weight. Use `tau_seconds` for a sensor\nthat reports on change, or for a stream that stops for periods.",
"properties": {
"alpha": {
"description": "The weight of the newest value, from 0 to 1.",
"format": "double",
"type": [
"number",
"null"
]
},
"half_life": {
"description": "The number of messages after which the weight of a value is half.",
"format": "double",
"type": [
"number",
"null"
]
},
"tau_seconds": {
"description": "The time constant in seconds. After this time, the weight of an old\nvalue is about 37%. It reads the `time` field of the transform.",
"format": "double",
"type": [
"number",
"null"
]
},
"type": {
"const": "ewma",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"description": "The median of the last `size` values, with this value included. It\nremoves a spike of one sample.",
"properties": {
"size": {
"description": "The number of values in the window.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"type": {
"const": "median",
"type": "string"
}
},
"required": [
"type",
"size"
],
"type": "object"
},
{
"description": "A Hampel filter. When the value is more than `threshold` robust\nstandard deviations from the median of the window, the median replaces\nit. Otherwise the value stays. Use it before `detect` to remove\noutliers.",
"properties": {
"size": {
"description": "The number of values in the window, with this value included.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"threshold": {
"description": "The number of scaled MADs from the median that makes an outlier.\nThe default is `3`.",
"format": "double",
"type": [
"number",
"null"
]
},
"type": {
"const": "hampel",
"type": "string"
}
},
"required": [
"type",
"size"
],
"type": "object"
},
{
"description": "A Savitzky–Golay filter. It fits a polynomial of degree `order` to the\nlast `size` values by least squares, and gives its value at the newest\npoint. It keeps the shape of peaks. The window uses only earlier\nvalues. Until the window holds more than `order` values, the value\npasses unchanged.",
"properties": {
"order": {
"description": "The degree of the polynomial. It must be less than `size`. The\ndefault is `2`.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"size": {
"description": "The number of values in the window, with this value included.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"type": {
"const": "savitzky_golay",
"type": "string"
}
},
"required": [
"type",
"size"
],
"type": "object"
}
]
},
"SmoothTransformConfig": {
"description": "Smooths a numeric field against the earlier values of its series. The\ntransform writes the result into the field, or under `as`.\n\nAll methods except `ewma` with `tau_seconds` use the order of the values,\nand not their time. Thus, `time` is permitted only with `ewma` and\n`tau_seconds`. With other methods, `time` is an error when the pipeline\nbuilds.",
"properties": {
"as": {
"description": "The field to write the smoothed value to. Leave it out to replace the\nvalue of `field`.",
"type": [
"string",
"null"
]
},
"field": {
"description": "The numeric field to smooth.",
"type": "string",
"x-message-field": true
},
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out for\none series.",
"items": {
"type": "string"
},
"type": "array"
},
"method": {
"$ref": "#/$defs/SmoothMethod",
"description": "The method to smooth the value with."
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have the field or a group\nfield. The default is `error`."
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"time": {
"description": "For `ewma` with `tau_seconds` only: the field that holds the time of\neach message, as an RFC 3339 string or as milliseconds since the epoch.\nLeave it out to use the arrival time.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"field",
"method"
],
"title": "smooth",
"type": "object"
},
"SplitterTransformConfig": {
"description": "Divides one batch into smaller batches.\n\nThe last batch contains the messages that remain and can be smaller. For\nexample, 4 messages with `out_size: 3` give a batch of 3 and a batch of 1.",
"properties": {
"out_size": {
"description": "The number of messages in each sent batch.",
"format": "uint",
"minimum": 0,
"type": "integer"
}
},
"required": [
"out_size"
],
"title": "splitter",
"type": "object"
},
"StringFilterOperatorKind": {
"description": "How a `string` condition compares a string to the value in the config.",
"enum": [
"equal_to",
"not_equal_to",
"contains"
],
"type": "string"
},
"ThrottleTransformConfig": {
"description": "Passes a maximum of one message per key in each period of `seconds`, and\ndrops the other messages.\n\nThe first message for each key passes. The next message that passes is the\nfirst one that arrives `seconds` or more after it. The transform drops all\nmessages between them. The period starts at the message that passed. It is\nnot aligned to the clock.\n\nThe transform does not keep messages to send later. A key that becomes\nquiet sends nothing until its next message. Use `resample` when you need the\nlast value of each period, or a value from a quiet key.\n\n`throttle` does not read a value. It passes complete messages with all\ntheir fields.",
"properties": {
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out for\none series.",
"items": {
"type": "string"
},
"type": "array"
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have a group field. The\ndefault is `error`."
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"seconds": {
"description": "The minimum time between two messages that pass for one key, in\nseconds.",
"format": "double",
"type": "number"
},
"time": {
"description": "The field that holds the time of each message, as an RFC 3339 string\nor as milliseconds since the epoch. Leave it out to use the arrival\ntime.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"seconds"
],
"title": "throttle",
"type": "object"
},
"TimeFormat": {
"description": "How a [`Mapping::TimeBucket`] writes a time.",
"oneOf": [
{
"const": "rfc3339",
"description": "An RFC 3339 string in UTC, with milliseconds. kayak writes all times in\nthis format.",
"type": "string"
},
{
"const": "millis",
"description": "Milliseconds since the epoch, as a number.",
"type": "string"
}
]
},
"TransformConfig": {
"oneOf": [
{
"$ref": "#/$defs/BufferTransformConfig",
"properties": {
"type": {
"const": "buffer",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/HttpTransformConfig",
"properties": {
"type": {
"const": "http",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/SplitterTransformConfig",
"properties": {
"type": {
"const": "splitter",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/ReduceTransformConfig",
"properties": {
"type": {
"const": "reducer",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/FilterTransformConfig",
"properties": {
"type": {
"const": "filter",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/RememberTransformConfig",
"properties": {
"type": {
"const": "remember",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/RecallTransformConfig",
"properties": {
"type": {
"const": "recall",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/MapTransformConfig",
"properties": {
"type": {
"const": "map",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/ScriptTransformConfig",
"properties": {
"type": {
"const": "script",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/DeadbandTransformConfig",
"properties": {
"type": {
"const": "deadband",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/ThrottleTransformConfig",
"properties": {
"type": {
"const": "throttle",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/PivotTransformConfig",
"properties": {
"type": {
"const": "pivot",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/DeriveTransformConfig",
"properties": {
"type": {
"const": "derive",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/RollingTransformConfig",
"properties": {
"type": {
"const": "rolling",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/SmoothTransformConfig",
"properties": {
"type": {
"const": "smooth",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/DetectTransformConfig",
"properties": {
"type": {
"const": "detect",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/ResampleTransformConfig",
"properties": {
"type": {
"const": "resample",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/FeaturesTransformConfig",
"properties": {
"type": {
"const": "features",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
}
],
"type": "object"
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "Run a draft's transforms over some messages.",
"properties": {
"buckets": {
"additionalProperties": {
"additionalProperties": true,
"type": "object"
},
"description": "what that bucket should already hold, keyed as the pipeline keys it.\nThe warm-up a stateful chain would otherwise have to be talked through.",
"type": "object"
},
"messages": {
"description": "the messages to put in, as one batch — which is what a `buffer` or a\n`reduce` will treat them as. A sample from `POST /api/inputs/sample` is\nwhat the UI puts here.",
"items": true,
"type": "array"
},
"state": {
"anyOf": [
{
"$ref": "#/$defs/PipelineState"
},
{
"type": "null"
}
],
"description": "the pipeline's `state` block, if its transforms need one. The bucket it\nnames is created for this request alone."
},
"transforms": {
"default": [],
"description": "the transforms, in order, exactly as they would be written in the\npipeline. An empty list is allowed and answers the trivial question.",
"items": {
"$ref": "#/$defs/TransformConfig"
},
"type": "array"
}
},
"required": [
"messages"
],
"title": "pipeline dry run request",
"type": "object"
}PipelineDryRunResponse
What the chain did.
schema
json
{
"$defs": {
"FailurePhase": {
"description": "Where a chain went wrong, when it did.",
"oneOf": [
{
"const": "build",
"description": "The transform could not be built — the same failure creating the\npipeline would have given, which is most of the value of building the\nreal thing.",
"type": "string"
},
{
"const": "apply",
"description": "It was built and failed on a message. A running pipeline would drop\nthat batch and carry on; a dry run stops, because what happened at the\npoint of failure is the thing being asked about.",
"type": "string"
}
]
},
"StageResult": {
"description": "What one transform in the chain did.",
"properties": {
"batches": {
"description": "what it handed on, one entry per batch. Several batches is a\n`splitter`; none is a `filter` that dropped everything, or a `buffer`\nstill holding what it was given.",
"items": {
"items": true,
"type": "array"
},
"type": "array"
},
"index": {
"description": "its position in `transforms`.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"kind": {
"description": "which transform it is (`filter`, `reduce`, ...), so the answer can be\nread without counting down the list.",
"type": "string"
},
"on_flush": {
"description": "what it handed on when the chain was drained at the end, if anything.\n\nKept apart from `batches` because the difference is the interesting\npart: a running pipeline releases these on a timer or a gate rather\nthan when the messages arrive, so a dry run that folded the two\ntogether would make a `buffer` look like it passes everything straight\nthrough.",
"items": {
"items": true,
"type": "array"
},
"type": "array"
}
},
"required": [
"index",
"kind",
"batches"
],
"type": "object"
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "What the chain did.",
"oneOf": [
{
"properties": {
"buckets": {
"additionalProperties": {
"additionalProperties": true,
"type": "object"
},
"description": "what the private buckets ended up holding, so a `remember` is\nvisible rather than being a side effect nobody can see.",
"type": "object"
},
"outcome": {
"const": "ran",
"type": "string"
},
"stages": {
"description": "one entry per transform, in order.",
"items": {
"$ref": "#/$defs/StageResult"
},
"type": "array"
}
},
"required": [
"outcome",
"stages"
],
"type": "object"
},
{
"properties": {
"at": {
"description": "which transform failed, by position.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"kind": {
"type": "string"
},
"message": {
"type": "string"
},
"outcome": {
"const": "failed",
"type": "string"
},
"phase": {
"$ref": "#/$defs/FailurePhase"
},
"stages": {
"description": "the stages that completed before it. Kept, rather than thrown away\nwith the failure: how far the messages got is half of what says\n*why* the failing transform failed.",
"items": {
"$ref": "#/$defs/StageResult"
},
"type": "array"
}
},
"required": [
"outcome",
"at",
"kind",
"phase",
"message"
],
"type": "object"
}
],
"title": "pipeline dry run response"
}PipelineDto
One pipeline as the API reports it: the id that it runs under, the config that kayak built it from, and the status of its run loop.
schema
json
{
"$defs": {
"AckMode": {
"description": "When an input acknowledges a message to its broker.",
"oneOf": [
{
"const": "on_receipt",
"description": "Acknowledge the message when it arrives, before the transforms and the\noutputs. The default. A crash before the output writes the message can\nlose it.",
"type": "string"
},
{
"const": "on_delivery",
"description": "Acknowledge the message when it leaves this pipeline. Each output of\nthis pipeline must return, with or without success. Each downstream\npipeline must accept the message into its queue. A failed output does\nnot stop the acknowledgement. kayak does not wait for the outputs of the\ndownstream pipelines.",
"type": "string"
}
]
},
"Aggregation": {
"description": "One value to calculate for a group, and the field name for the result.",
"properties": {
"as": {
"description": "The field that contains the result in the sent message. Each\naggregation must have a different `as`. It must not be the same as a\n`group_by` field.",
"type": "string"
},
"field": {
"description": "The field to aggregate. Required for all functions except `count`.\nWithout a `field`, `count` counts the messages.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"function": {
"$ref": "#/$defs/ReduceFnKind",
"description": "How to combine the values."
}
},
"required": [
"function",
"as"
],
"title": "aggregation",
"type": "object"
},
"ArithmeticOperator": {
"description": "What an [`Mapping::Arithmetic`] does with its two operands.",
"oneOf": [
{
"const": "add",
"description": "Left + right.",
"type": "string"
},
{
"const": "subtract",
"description": "Left − right.",
"type": "string"
},
{
"const": "multiply",
"description": "Left × right.",
"type": "string"
},
{
"const": "divide",
"description": "Left ÷ right. A literal zero on the right is an error when the pipeline\nbuilds. For a field that holds zero, `on_zero` applies.",
"type": "string"
},
{
"const": "min",
"description": "The smaller of left and right. With a literal on one side, this is an\nupper limit.",
"type": "string"
},
{
"const": "max",
"description": "The larger of left and right. With a literal on one side, this is a\nlower limit.",
"type": "string"
}
]
},
"Band": {
"description": "The power in one frequency band, as a feature.",
"properties": {
"as": {
"description": "The field to write the power of the band to.",
"type": "string"
},
"high_hz": {
"description": "The upper limit of the band, in hertz. The band does not include this\nvalue. It must be more than `low_hz`.",
"format": "double",
"type": "number"
},
"low_hz": {
"description": "The lower limit of the band, in hertz. The band includes this value.",
"format": "double",
"type": "number"
}
},
"required": [
"low_hz",
"high_hz",
"as"
],
"title": "band",
"type": "object"
},
"BufferConfig": {
"description": "How an input collects its messages into batches before the transforms.\n\nThe three types use two limits: a count, a time, or both. With both, the\nfirst limit that is reached closes the batch. A buffer never sends an empty\nbatch. The time starts when the first message of a batch arrives, so a\nquiet input sends nothing.\n\n`size` is a minimum, not a maximum. The buffer does not divide a batch\nthat arrives. Thus an input with `max_batch` can give a larger batch.",
"oneOf": [
{
"description": "Wait for a number of messages. There is no time limit.",
"properties": {
"size": {
"description": "The number of messages in a batch.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"type": {
"const": "static",
"type": "string"
}
},
"required": [
"type",
"size"
],
"type": "object"
},
{
"description": "Wait for a time. The batch contains at least one message. The time\nstarts when the first message arrives.",
"properties": {
"type": {
"const": "tumbling",
"type": "string"
},
"window_seconds": {
"description": "The time to collect messages, in s, from the first message.",
"format": "uint",
"minimum": 0,
"type": "integer"
}
},
"required": [
"type",
"window_seconds"
],
"type": "object"
},
{
"description": "Use both limits. The first limit that is reached closes the batch. Use\nthis type when the rate of the input changes. `size` sets the largest\nbatch when the input is busy. `window_seconds` sets the longest wait\nwhen the input is quiet.",
"properties": {
"size": {
"description": "The number of messages that closes the batch immediately.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"type": {
"const": "batch",
"type": "string"
},
"window_seconds": {
"description": "The maximum time to wait, in s, from the first message in the batch.",
"format": "uint",
"minimum": 0,
"type": "integer"
}
},
"required": [
"type",
"size",
"window_seconds"
],
"type": "object"
}
]
},
"BufferGateConfig": {
"description": "A condition on a state bucket that releases a `buffer` transform.\n\nThe buffer tests the conditions against the bucket entry as an object. The\nnames that `remember` wrote are its fields. `field` is a dotted path, as in\nall other transforms. All conditions must be true.\n\nThe gate applies to the full buffer. It does not test each kept message.\nWhen the gate opens, the buffer sends all kept messages.",
"properties": {
"bucket": {
"description": "The bucket to watch. The default is the bucket in the `state` of this\npipeline. A pipeline with no `state` must set it.",
"type": [
"string",
"null"
]
},
"conditions": {
"description": "The conditions that must all be true to release the buffer. At least\none condition is required.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"key": {
"description": "The key in the bucket to read. This is a literal key, not a field path.\nIf you do not set it, the gate reads the value for the full bucket.\n`remember` writes that value when the `state` of its pipeline has no\n`key`.",
"type": [
"string",
"null"
]
}
},
"required": [
"conditions"
],
"title": "buffer gate",
"type": "object"
},
"BufferTransformConfig": {
"description": "Keeps messages and sends them on when a trigger fires.\n\nThere are three triggers: a message count, a time, and a condition on a\nstate bucket. You can use them together. The first trigger that fires\nreleases the messages. A buffer with no trigger fails to build.\n\n`size` sends batches of exactly that number of messages. `seconds` and\n`until` send all messages that the buffer keeps, as one batch.\n\nThe `buffer` setting on an input is a different thing. It makes batches\nbefore the transforms. The `buffer` transform makes batches at its\nposition in the chain, for example after a `filter` or a `recall`.",
"properties": {
"max_messages": {
"description": "The maximum number of messages to keep. At this number, the buffer sends\nall kept messages and writes one warning to the log. Required if `size`\nis not set. Without a limit, a condition that is never true makes the\nbuffer use more and more memory.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"seconds": {
"description": "Send all kept messages this number of seconds after the first kept\nmessage. The time starts when the buffer keeps a message. An empty\nbuffer sends nothing.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"size": {
"description": "Send batches of exactly this number of messages, when each batch is\nfull.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"until": {
"anyOf": [
{
"$ref": "#/$defs/BufferGateConfig"
},
{
"type": "null"
}
],
"description": "Send all kept messages when a condition on a state bucket is true.\nBuckets are global, so a different pipeline can write the value that\nopens the gate."
}
},
"title": "buffer",
"type": "object"
},
"CastType": {
"description": "The type that a `cast` converts a value to.\n\nThese types are not the column types of the database outputs. There is no\n`bigint` and no `decimal`. Here, `json` parses a string that contains JSON.",
"oneOf": [
{
"const": "text",
"description": "A string. A number or a boolean is written as JSON writes it. An object\nor an array is an error.",
"type": "string"
},
{
"const": "integer",
"description": "A whole number. A string is parsed. A number with a fractional part is\nan error. The cast does not round.",
"type": "string"
},
{
"const": "float",
"description": "A number. A string is parsed.",
"type": "string"
},
{
"const": "boolean",
"description": "True or false. The cast accepts the strings `true` and `false` in\nuppercase or lowercase, and the numbers 1 and 0. Other values are an\nerror.",
"type": "string"
},
{
"const": "timestamp",
"description": "A timestamp, written as RFC 3339. A string is parsed and written again\nin one format. A number is read as **seconds** since the epoch, with\nfractions. The column mapping reads a number in the same way.",
"type": "string"
},
{
"const": "date",
"description": "A calendar date, written as `2026-08-10`. A string can be a date or a\nfull timestamp. From a timestamp, the cast uses the date.",
"type": "string"
},
{
"const": "uuid",
"description": "A UUID, in lowercase. The cast accepts only a string in the canonical\nform with hyphens.",
"type": "string"
},
{
"const": "json",
"description": "Parses the JSON that a string contains. The input must be a string. Use\nit for a payload that is encoded inside another payload.",
"type": "string"
}
]
},
"ClickhouseInputConfig": {
"description": "Reads a `ClickHouse` table, view or query at an interval and sends each row\nas a message. It uses the HTTP interface of `ClickHouse`.\n\n`ClickHouse` sends the rows as `JSONEachRow`. A `DateTime` is ISO 8601, an\n`Int64` is a number, and a `Decimal` keeps its digits. The modes, the\nwatermark and the limits of an incremental read are the same as on the\n`postgres` input.",
"properties": {
"columns": {
"description": "The columns to read, in order. Leave it empty to read all columns. For\nan incremental input, the list must include `field`.",
"items": {
"type": "string"
},
"type": "array"
},
"connection": {
"description": "The name of the clickhouse connection to read through. Declare the\nconnection in the connections file.",
"type": "string",
"x-connection": "clickhouse"
},
"interval_secs": {
"description": "The time between two reads, in seconds. The time starts at the end of a\nread. Thus, two reads never overlap. The first read occurs when the\npipeline starts.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"max_batch": {
"description": "The maximum number of rows in one batch. The default is 1. The input\nputs rows that it already read into a batch. It does not wait for more\nrows to fill a batch.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"mode": {
"$ref": "#/$defs/PollMode",
"description": "Whether each read returns all rows (`snapshot`) or only the rows after\nthe last read (`incremental`)."
},
"page_size": {
"description": "The maximum number of rows that one query returns. The default is\n1000. When a page is full, the input reads the next page immediately.\nThe interval starts after a page that is not full. A `snapshot` ignores\nthis field and reads all rows in one query.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"query": {
"description": "A `SELECT` to read in place of a table. The input puts the query in a\nsubquery. It adds the cursor condition, the `ORDER BY` and the page\nlimit outside the subquery. Thus, the query needs no placeholder and no\n`ORDER BY`.\n\nWrite one statement with no semicolon at the end. You can use all SQL\nthat the server accepts in a subquery, for example `WITH`.",
"type": [
"string",
"null"
]
},
"table": {
"description": "The table or view to read, as `name` or `schema.name`. Give exactly one\nof `table` and `query`.",
"type": [
"string",
"null"
]
}
},
"required": [
"connection",
"interval_secs",
"mode"
],
"title": "clickhouse",
"type": "object"
},
"ClickhouseOutputConfig": {
"description": "Inserts each batch into a `ClickHouse` table, one insert for each batch.\n\n`columns` has the same format as on the `postgres` output. Each entry names\na column, its type and the field to read. The default `field` is the name of\nthe column. Without `columns`, the table has a `payload` column that\ncontains each message as JSON text.\n\n`ClickHouse` has no auto-increment column and no unique constraint.\n`order_by` names the sorting key of the `MergeTree` table. If you do not set\nit, the table gets a `received_at` timestamp and is sorted by it. A sorting\nkey does not remove duplicate rows.\n\nThe output creates the table if it does not exist. Set `create_table` to\nfalse for a table that another system owns. The output does not change an\nexisting table.",
"properties": {
"columns": {
"description": "The column for each message field. If you do not set it, the output\nkeeps each full message as JSON text in a `payload` column.",
"items": {
"$ref": "#/$defs/ColumnMapping"
},
"type": "array"
},
"connection": {
"description": "The name of the clickhouse connection to insert through. Declare the\nconnection in the connections file. The connection gives the url, the\ndatabase and the user.",
"type": "string",
"x-connection": "clickhouse"
},
"create_table": {
"description": "Create the table on start if it does not exist. The default is true.",
"type": [
"boolean",
"null"
]
},
"on_extra_fields": {
"$ref": "#/$defs/ExtraFieldPolicy",
"description": "What to do with a message that has fields that no column reads."
},
"order_by": {
"description": "The columns that sort the created table. This is the sorting key of the\n`MergeTree` table and its index. If you do not set it, the table gets a\n`received_at` timestamp and is sorted by it. The output makes these\ncolumns `NOT NULL`, because `ClickHouse` cannot sort by a nullable key.",
"items": {
"type": "string"
},
"type": "array"
},
"table": {
"description": "The table to insert into. The output creates it if it does not exist.\nYou can add a database (`analytics.readings`). This database replaces\nthe database of the connection. Use only letters, digits and\nunderscores.",
"type": "string"
}
},
"required": [
"connection",
"table"
],
"title": "clickhouse",
"type": "object"
},
"ColumnMapping": {
"description": "One message field that the output writes to one column.\n\nThe default of `field` is `name`. When the message uses the column names,\ngive only `name` and `type`. `field` is a dotted path, for example\n`_meta.subject`. A key that contains a dot and matches exactly has\npriority over the path.",
"properties": {
"field": {
"description": "The field to read, as a dotted path. The default is the name of the\ncolumn.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"message": {
"description": "Write the full message to this column. Use it only with a `json`\ncolumn. Do not use it with `field`.",
"type": "boolean"
},
"name": {
"description": "The name of the column in the table. Use only letters, digits and\nunderscores.",
"type": "string"
},
"nullable": {
"description": "Whether the column accepts `NULL`. The default is true. With `false`,\nthe output makes the column `NOT NULL`, and a missing field is an\nerror.",
"type": [
"boolean",
"null"
]
},
"on_missing": {
"anyOf": [
{
"$ref": "#/$defs/MissingColumnPolicy"
},
{
"type": "null"
}
],
"description": "What to do with a message that does not have the field. The default is\n`null`, or `error` for a column that is not nullable."
},
"type": {
"$ref": "#/$defs/ColumnType",
"description": "The type of the column. The output checks each value against it."
}
},
"required": [
"name",
"type"
],
"title": "column",
"type": "object"
},
"ColumnType": {
"description": "The type of a column. The names are the same for all database outputs.\nEach output changes them into the types of its server.\n\nThe output checks each value against the type before it sends the value.\nIt does not convert values. For example, the string `\"12.5\"` in a `float`\ncolumn is an error. To convert a value, use a `cast` in a `map` transform.",
"oneOf": [
{
"const": "text",
"description": "A string of any length. Only a JSON string is accepted.",
"type": "string"
},
{
"const": "integer",
"description": "A 32-bit whole number. A JSON number with a fractional part is an\nerror. A number outside the range is an error.",
"type": "string"
},
{
"const": "bigint",
"description": "A 64-bit whole number.",
"type": "string"
},
{
"const": "float",
"description": "A double-precision floating point number.",
"type": "string"
},
{
"const": "decimal",
"description": "An exact decimal. The output sends the digits as they are in the\nmessage, so no precision is lost.",
"type": "string"
},
{
"const": "boolean",
"description": "True or false. Only a JSON boolean is accepted.",
"type": "string"
},
{
"const": "timestamp",
"description": "A date and time with a time zone. The server parses a JSON string as\nISO 8601 or RFC 3339. The output reads a JSON number as **seconds**\nsince the epoch, with fractions.",
"type": "string"
},
{
"const": "date",
"description": "A calendar date, as a JSON string (`2026-08-10`).",
"type": "string"
},
{
"const": "uuid",
"description": "A UUID, as a JSON string.",
"type": "string"
},
{
"const": "json",
"description": "Any JSON value, stored as JSON.",
"type": "string"
}
]
},
"ConcatPart": {
"description": "One piece of a [`Mapping::Concat`].",
"oneOf": [
{
"description": "A value from the message. A string is used as it is. A number or a\nboolean is written as JSON writes it. An object or an array is an error.",
"properties": {
"field": {
"description": "The field to read.",
"type": "string"
},
"type": {
"const": "field",
"type": "string"
}
},
"required": [
"type",
"field"
],
"type": "object"
},
{
"description": "Literal text, for example a separator, a prefix or a suffix.",
"properties": {
"type": {
"const": "value",
"type": "string"
},
"value": {
"description": "The text.",
"type": "string"
}
},
"required": [
"type",
"value"
],
"type": "object"
}
]
},
"Condition": {
"description": "One test on a message.\n\n`filter`, `remember` and the gate of a `buffer` use conditions. A list of\nconditions means that all of them must match. There is no `or` and no\nnesting. Use `invert` on a `filter`, or `none_of`, for a negative test.",
"oneOf": [
{
"description": "Compares a field to a number. A message whose field is missing or is not\na number does not match.",
"properties": {
"field": {
"description": "The field to test, as a dotted path.",
"type": "string",
"x-message-field": true
},
"operator": {
"$ref": "#/$defs/NumericFilterOperatorKind",
"description": "How to compare the field."
},
"type": {
"const": "numeric",
"type": "string"
},
"value": {
"description": "The number to compare to.",
"format": "double",
"type": "number"
}
},
"required": [
"type",
"field",
"operator",
"value"
],
"type": "object"
},
{
"description": "Compares a field to a string. A message whose field is missing or is not\na string does not match.",
"properties": {
"field": {
"description": "The field to test, as a dotted path.",
"type": "string",
"x-message-field": true
},
"operator": {
"$ref": "#/$defs/StringFilterOperatorKind",
"description": "How to compare the field."
},
"type": {
"const": "string",
"type": "string"
},
"value": {
"description": "The string to compare to.",
"type": "string"
}
},
"required": [
"type",
"field",
"operator",
"value"
],
"type": "object"
},
{
"description": "Matches when the field is a string equal to one of `values`. A message\nwhose field is missing or is not a string does not match.",
"properties": {
"field": {
"description": "The field to test, as a dotted path.",
"type": "string",
"x-message-field": true
},
"type": {
"const": "one_of",
"type": "string"
},
"values": {
"description": "The strings that match.",
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"type",
"field",
"values"
],
"type": "object"
},
{
"description": "Matches when the field is a string equal to none of `values`. A message\nwhose field is missing or is not a string also does not match.",
"properties": {
"field": {
"description": "The field to test, as a dotted path.",
"type": "string",
"x-message-field": true
},
"type": {
"const": "none_of",
"type": "string"
},
"values": {
"description": "The strings that do not match.",
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"type",
"field",
"values"
],
"type": "object"
}
]
},
"Config": {
"description": "One pipeline. kayak merges all inputs into one stream. The stream goes\nthrough the transforms in sequence. Each batch that results goes to each\noutput.",
"properties": {
"id": {
"type": [
"string",
"null"
]
},
"inputs": {
"description": "The inputs of the pipeline. At least one input is required. Batches\narrive in the sequence in which the inputs make them. There is no\nsequence between two different inputs.",
"items": {
"$ref": "#/$defs/InputConfig"
},
"type": "array"
},
"outputs": {
"default": [],
"description": "The outputs. Optional. A pipeline that only sends to downstream\npipelines needs no output.",
"items": {
"$ref": "#/$defs/OutputConfig"
},
"type": "array"
},
"state": {
"anyOf": [
{
"$ref": "#/$defs/PipelineState"
},
{
"type": "null"
}
],
"description": "The state bucket of this pipeline, and the key of its messages. Required\nfor `remember`, `recall` and the streaming transforms that keep state,\nfor example `rolling`. These transforms fail to build without it."
},
"transforms": {
"default": [],
"description": "The transforms, in sequence. Optional. A pipeline that only moves\nmessages needs no transform.",
"items": {
"$ref": "#/$defs/TransformConfig"
},
"type": "array"
}
},
"required": [
"inputs"
],
"type": "object"
},
"DeadbandMode": {
"description": "The unit of the `delta` of a deadband: an amount, or a percentage of the\nlast value that passed.",
"oneOf": [
{
"const": "absolute",
"description": "`delta` is in the units of the field. For example, `0.5` is half a\ndegree.",
"type": "string"
},
{
"const": "percent",
"description": "`delta` is a percentage of the last value that passed. For example,\n`2` is two percent. When the last value is zero, every message passes.",
"type": "string"
}
]
},
"DeadbandTransformConfig": {
"description": "Drops a message when its field did not change sufficiently since the last\nmessage that passed. This is a filter that keeps state per key.\n\nThe first message for each key always passes. After that, a message passes\nin two cases:\n\n- `field` differs from the last value that passed by more than `delta`.\n- `max_seconds` went by since the last message passed.\n\nWith `flatline_seconds`, the transform also finds a stuck sensor. When the\nvalue does not change for that time, the next message passes with\n`stuck: true`. This occurs one time for each flat period.",
"properties": {
"delta": {
"description": "The change that a message needs to pass. The unit is set by `mode`.",
"format": "double",
"type": "number"
},
"field": {
"description": "The numeric field to compare.",
"type": "string",
"x-message-field": true
},
"flatline_seconds": {
"description": "After this number of seconds with no change, pass the next message with\n`stuck: true`. This occurs one time for each flat period.",
"format": "double",
"type": [
"number",
"null"
]
},
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out for\none series.",
"items": {
"type": "string"
},
"type": "array"
},
"max_seconds": {
"description": "Pass a message when this number of seconds went by since the last\nmessage passed, also if the value did not change.",
"format": "double",
"type": [
"number",
"null"
]
},
"mode": {
"$ref": "#/$defs/DeadbandMode",
"description": "The unit of `delta`. The default is `absolute`."
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have the field or a group\nfield. The default is `error`."
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"time": {
"description": "The field that holds the time of each message, as an RFC 3339 string\nor as milliseconds since the epoch. Leave it out to use the arrival\ntime.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"field",
"delta"
],
"title": "deadband",
"type": "object"
},
"Derivation": {
"description": "One derived value and the field it is written to.",
"properties": {
"as": {
"description": "The field to write the result to.",
"type": "string"
},
"field": {
"description": "The numeric field to read.",
"type": "string",
"x-message-field": true
},
"function": {
"$ref": "#/$defs/DeriveFnKind",
"description": "The calculation to do."
},
"wrap_at": {
"description": "For `counter`: the value at which the counter goes back to zero. With\nit, the transform reads a decrease as a wrap. Without it, the\ntransform reads a decrease as a reset.",
"format": "double",
"type": [
"number",
"null"
]
}
},
"required": [
"function",
"field",
"as"
],
"title": "derivation",
"type": "object"
},
"DeriveFnKind": {
"description": "How the transform calculates a value from this message and the previous\nmessage.",
"oneOf": [
{
"const": "rate",
"description": "The change per second since the previous message, from the `time`\nfield. The value is `null` for the first message and when no time went\nby.",
"type": "string"
},
{
"const": "delta",
"description": "The change since the previous message. The value is `null` for the\nfirst message.",
"type": "string"
},
{
"const": "cumsum",
"description": "The running total of the field, from the first message.",
"type": "string"
},
{
"const": "counter",
"description": "The running total of the increases. Use it for a counter that resets\nor wraps. When the value decreases and `wrap_at` is set, the transform\nadds the increase through `wrap_at`. When `wrap_at` is not set, it adds\nthe new value.",
"type": "string"
}
]
},
"DeriveTransformConfig": {
"description": "Writes a value onto each message that needs the previous message of the\nsame key. The value can be a rate of change, a delta, a running total or a\ncounter that wraps.\n\nYou can give many derivations. The transform writes each one to its own\n`as` field. The first message for each key has no previous message. For\nthat message, `rate` and `delta` write `null`.",
"properties": {
"derive": {
"description": "The values to calculate. Give one or more, each with a different\n`as`.",
"items": {
"$ref": "#/$defs/Derivation"
},
"type": "array"
},
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out for\none series.",
"items": {
"type": "string"
},
"type": "array"
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have a derived field or a\ngroup field. The default is `error`."
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"time": {
"description": "The field that holds the time of each message, as an RFC 3339 string\nor as milliseconds since the epoch. Leave it out to use the arrival\ntime.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"derive"
],
"title": "derive",
"type": "object"
},
"DetectLearn": {
"description": "The readings that a `detect` baseline learns from, for the methods that\ncontinue to learn.",
"oneOf": [
{
"const": "all",
"description": "All readings, with anomalies included. A change that continues becomes\nthe new normal at the speed of the baseline.",
"type": "string"
},
{
"const": "normal_only",
"description": "Only the readings that are not anomalies. An anomaly cannot change the\nbaseline. Use it with `readapt_after_seconds`. Without it, the\ntransform flags a real change of level permanently.",
"type": "string"
}
]
},
"DetectMethod": {
"description": "How the transform finds an anomaly.\n\nThe window methods (`zscore`, `mad`) compare a value with the values before\nit. The window does not include the value, so a spike does not change its\nown baseline. The chart methods (`cusum` without `target`, `ewma_chart`,\n`western_electric`) fix their baseline at the end of the warm-up.",
"oneOf": [
{
"description": "An anomaly is more than `threshold` standard deviations from the mean\nof the last `size` values.",
"properties": {
"size": {
"description": "The number of earlier values in the baseline.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"threshold": {
"description": "The number of standard deviations that makes an anomaly. The\ndefault is `3`.",
"format": "double",
"type": [
"number",
"null"
]
},
"type": {
"const": "zscore",
"type": "string"
}
},
"required": [
"type",
"size"
],
"type": "object"
},
{
"description": "An anomaly is more than `threshold` scaled median absolute deviations\nfrom the median of the last `size` values. Use it when the baseline\ncontains outliers.",
"properties": {
"size": {
"description": "The number of earlier values in the baseline.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"threshold": {
"description": "The number of scaled MADs that makes an anomaly. The default is\n`3.5`.",
"format": "double",
"type": [
"number",
"null"
]
},
"type": {
"const": "mad",
"type": "string"
}
},
"required": [
"type",
"size"
],
"type": "object"
},
{
"description": "A two-sided CUSUM. It adds the drift above `target` and the drift below\n`target` in two sums. An anomaly is a sum that is more than\n`threshold`. Then that sum goes back to zero. Use it to find a small\nshift that continues.",
"properties": {
"drift": {
"description": "The change per message that does not count as drift, in the units\nof the field.",
"format": "double",
"type": "number"
},
"target": {
"description": "The expected value of the series. Leave it out to use the mean of\nthe warm-up.",
"format": "double",
"type": [
"number",
"null"
]
},
"threshold": {
"description": "The sum of drift that makes an anomaly.",
"format": "double",
"type": "number"
},
"type": {
"const": "cusum",
"type": "string"
}
},
"required": [
"type",
"drift",
"threshold"
],
"type": "object"
},
{
"description": "An EWMA control chart. An anomaly is a smoothed value outside a band of\n`threshold` standard deviations around the mean of the warm-up. It\nfinds small shifts and ignores single points.",
"properties": {
"alpha": {
"description": "The weight of the newest value, from 0 to 1. The default is `0.2`.",
"format": "double",
"type": [
"number",
"null"
]
},
"threshold": {
"description": "The width of the band, in standard deviations. The default is `3`.",
"format": "double",
"type": [
"number",
"null"
]
},
"type": {
"const": "ewma_chart",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"description": "The Western Electric rules, against the mean and deviation of the\nwarm-up:\n\n- one point more than 3σ from the mean,\n- two of three points more than 2σ from the mean on one side,\n- four of five points more than 1σ from the mean on one side,\n- eight points in a sequence on one side.\n\nThe transform writes the rule that matched beside the flag.",
"properties": {
"type": {
"const": "western_electric",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"description": "An anomaly is `size` identical values in a sequence. Use it to find a\nstuck instrument.",
"properties": {
"size": {
"description": "The number of identical values in a sequence that makes an anomaly.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"type": {
"const": "flatline",
"type": "string"
}
},
"required": [
"type",
"size"
],
"type": "object"
},
{
"description": "An anomaly is more than `threshold` deviations from a baseline that\ncontinues to learn. The baseline is an exponentially weighted mean and\nspread, each with its own time constant. Use it for a series that\ndrifts slowly and arrives at irregular times. It reads the `time` field\nof the transform.",
"properties": {
"mean_tau_seconds": {
"description": "The time constant of the mean, in seconds. A smaller value makes the\nmean follow the series more quickly.",
"format": "double",
"type": "number"
},
"min_spread": {
"description": "The minimum deviation, in the units of the field. Without it, a\nsignal that was very quiet flags its first small change. The default\nis `0`.",
"format": "double",
"type": [
"number",
"null"
]
},
"spread_tau_seconds": {
"description": "The time constant of the spread, in seconds. Make it longer than\n`mean_tau_seconds`, so that a short period of noise does not make\nthe band wider immediately.",
"format": "double",
"type": "number"
},
"threshold": {
"description": "The number of deviations that makes an anomaly. The default is `3`.",
"format": "double",
"type": [
"number",
"null"
]
},
"type": {
"const": "ewma",
"type": "string"
}
},
"required": [
"type",
"mean_tau_seconds",
"spread_tau_seconds"
],
"type": "object"
}
]
},
"DetectMode": {
"description": "Which messages the `detect` transform sends.",
"oneOf": [
{
"const": "annotate",
"description": "Every message passes, with the flag and the score.",
"type": "string"
},
{
"const": "only_anomalies",
"description": "Only the anomalies pass, with the flag and the score.",
"type": "string"
}
]
},
"DetectTransformConfig": {
"description": "Flags anomalies in a numeric field against its own series. `method` selects\nhow the transform finds an anomaly.\n\nThe transform writes a boolean under `as`. The default is `anomaly`. It also\nwrites a score under `<as>_score`. The score is the distance from normal, in\nthe units of the method. A `filter` after the transform can use a stricter\nlimit on the score.\n\nThe warm-up is `min_samples` messages for each key. The transform flags\nnothing during the warm-up.",
"properties": {
"as": {
"description": "The field to write the flag to. The score goes to `<as>_score`. The\ndefault is `anomaly`.",
"type": [
"string",
"null"
]
},
"field": {
"description": "The numeric field to examine.",
"type": "string",
"x-message-field": true
},
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out for\none series.",
"items": {
"type": "string"
},
"type": "array"
},
"learn": {
"$ref": "#/$defs/DetectLearn",
"description": "For `zscore`, `mad` and `ewma`: the readings that the baseline learns\nfrom. The default is `all`."
},
"method": {
"$ref": "#/$defs/DetectMethod",
"description": "How the transform finds an anomaly."
},
"min_samples": {
"description": "The number of messages for each key before the transform flags\nanything. The default is the `size` of the method, or 30 for a method\nwith no `size`.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"mode": {
"$ref": "#/$defs/DetectMode",
"description": "Which messages to send. The default is `annotate`."
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have the field or a group\nfield. The default is `error`."
},
"readapt_after_seconds": {
"description": "With `learn: normal_only`: when readings are anomalies for this number\nof seconds without a break, the baseline learns from them. A change\nthat continues then becomes the new normal.",
"format": "double",
"type": [
"number",
"null"
]
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"time": {
"description": "The field that holds the time of each message, as an RFC 3339 string\nor as milliseconds since the epoch. The `ewma` method and\n`readapt_after_seconds` use it. Leave it out to use the arrival time.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"with_baseline": {
"description": "Also write the baseline of the reading, in the units of the field. The\nnormal value goes to `<as>_expected`. The permitted distance from it\ngoes to `<as>_band`. The value is `null` when the method has no\nbaseline.",
"type": "boolean"
}
},
"required": [
"field",
"method"
],
"title": "detect",
"type": "object"
},
"DummyConfig": {
"description": "Sends one generated message at a fixed interval. Use it to test a pipeline\nwithout a real source.\n\nEach message contains a `value` and the `current_time` at which the input\nsent it. The `payload` field sets the type of `value`. It can be a number\nfrom a sine wave or a random sentence.",
"properties": {
"amplitude": {
"description": "The peak of the sine wave. The value goes from `-amplitude` to\n`+amplitude`. Applies only to the `number` payload. The default is 1.",
"format": "double",
"type": [
"number",
"null"
]
},
"duration": {
"description": "The time between two messages, in s.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"payload": {
"anyOf": [
{
"$ref": "#/$defs/DummyPayload"
},
{
"type": "null"
}
],
"description": "The type of `value` in each message. `number` is a number from a sine\nwave. `text` is a random sentence. The default is `number`."
},
"period": {
"description": "The time for one full cycle of the sine wave, in s. Applies only to the\n`number` payload. The default is 60. The input samples the wave by the\nclock, so the period does not change with `duration`.",
"format": "double",
"type": [
"number",
"null"
]
}
},
"required": [
"duration"
],
"title": "dummy",
"type": "object"
},
"DummyPayload": {
"description": "The type of `value` in each message from a `dummy` input.",
"oneOf": [
{
"const": "number",
"description": "A number from a sine wave.",
"type": "string"
},
{
"const": "text",
"description": "A random sentence.",
"type": "string"
}
]
},
"EnvelopeConfig": {
"description": "How an input adds metadata about the source of each message.\n\nThe \"metadata\" section of each input lists its metadata. Examples are the\nsubject of a nats message and the topic, partition and offset of a kafka\nrecord. The metadata also contains the pipeline and the input type.\n\nThe input adds the metadata as ordinary fields on the message. Thus each\ntransform can use it as it uses the fields of the payload, for example\n`\"group_by\": [\"_meta.subject\"]`.\n\nIf you do not set an envelope, the input sends each message without\nchanges. An envelope changes the shape of each message from the input.\nUpdate the field paths downstream when you add one.",
"oneOf": [
{
"description": "Add the metadata as one more field on the message. The fields of the\npayload do not move.\n\nThis type works only on a payload that is a JSON object. The input\nskips a message that is a number or a string and writes a warning to\nthe log. Use `wrap` for these payloads.",
"properties": {
"meta": {
"description": "The field for the metadata object. The default is `_meta`.",
"type": [
"string",
"null"
]
},
"type": {
"const": "merge",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"description": "Put the full payload under a field, next to the metadata:\n`{\"value\": …, \"_meta\": {…}}`.\n\nThis type works with all payloads, for example a `1` or a `\"recipe-a\"`.\nEach field path downstream must then start with the payload field, for\nexample `value.temperature`.",
"properties": {
"meta": {
"description": "The field for the metadata object. The default is `_meta`.",
"type": [
"string",
"null"
]
},
"payload": {
"description": "The field for the original payload. The default is `value`.",
"type": [
"string",
"null"
]
},
"type": {
"const": "wrap",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
}
]
},
"ExtraFieldPolicy": {
"description": "What to do with a message that has fields that no column reads.",
"oneOf": [
{
"const": "ignore",
"description": "Write the mapped columns and ignore the other fields. This is the\ndefault.",
"type": "string"
},
{
"const": "error",
"description": "Fail the batch. Use it for a stream with a fixed shape, where a new\nfield is a problem.",
"type": "string"
}
]
},
"FeatureKind": {
"description": "A number that describes a window of readings.",
"oneOf": [
{
"const": "mean",
"description": "The arithmetic mean.",
"type": "string"
},
{
"const": "std",
"description": "The population standard deviation.",
"type": "string"
},
{
"const": "min",
"description": "The smallest value.",
"type": "string"
},
{
"const": "max",
"description": "The largest value.",
"type": "string"
},
{
"const": "range",
"description": "The largest value minus the smallest value.",
"type": "string"
},
{
"const": "slope",
"description": "The least-squares slope. It is per second with the `time` field, and\nper message without it.",
"type": "string"
},
{
"const": "skew",
"description": "The skewness: the direction of the longer tail.",
"type": "string"
},
{
"const": "kurtosis",
"description": "The excess kurtosis: the weight of the tails.",
"type": "string"
},
{
"const": "rms",
"description": "The root mean square.",
"type": "string"
},
{
"const": "crest_factor",
"description": "The peak magnitude divided by the RMS.",
"type": "string"
},
{
"const": "zero_crossings",
"description": "The number of times that the signal crosses zero.",
"type": "string"
},
{
"const": "n_peaks",
"description": "The number of local maxima.",
"type": "string"
},
{
"const": "autocorr1",
"description": "The autocorrelation at lag one.",
"type": "string"
},
{
"const": "dominant_frequency",
"description": "The strongest frequency above DC, in hertz. It needs a sample rate from\nthe `time` field or from `sample_rate_hz`.",
"type": "string"
},
{
"const": "count",
"description": "The number of readings in the window.",
"type": "string"
},
{
"const": "duration",
"description": "The time from the first reading to the last reading, in seconds, from\nthe `time` field.",
"type": "string"
}
]
},
"FeaturesTransformConfig": {
"description": "Changes a batch of readings into one message of features for each group.\nUse it to send a small set of numbers to a model, in place of the raw\nreadings. Put a `buffer` on the input. Without it, each batch has only one\nreading.\n\nThe transform writes:\n\n- each feature in `include` under its own name, for example `mean`, `rms`\n or `crest_factor`,\n- each entry in `bands` under its `as`,\n- the `group_by` fields under their leaf names, as `reduce` does.\n\nA feature with no value for the window is `null`. For example, a slope of\none point is `null`. The spectral features (`dominant_frequency`, `bands`)\nneed a sample rate. kayak uses `sample_rate_hz`, or calculates the rate\nfrom the `time` field. Without one of the two, the pipeline does not build.\nThis transform keeps no state and needs no state bucket.",
"properties": {
"bands": {
"description": "The frequency bands to calculate the power of. The transform writes\neach one under its `as`. Give `include`, `bands` or both.",
"items": {
"$ref": "#/$defs/Band"
},
"type": "array"
},
"field": {
"description": "The numeric field to read.",
"type": "string",
"x-message-field": true
},
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out to use\nthe full batch as one window.",
"items": {
"type": "string"
},
"type": "array"
},
"include": {
"description": "The features to calculate. The transform writes each one under its own\nname.",
"items": {
"$ref": "#/$defs/FeatureKind"
},
"type": "array"
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a reading that does not have the field or a group\nfield. The default is `error`."
},
"sample_rate_hz": {
"description": "The sample rate of the readings in hertz, for the spectral features.\nIt must be more than zero. kayak uses it in place of the rate from\n`time`. Use it when the source has timestamps with low resolution.",
"format": "double",
"type": [
"number",
"null"
]
},
"time": {
"description": "The field that holds the time of each reading, as an RFC 3339 string or\nas milliseconds since the epoch. `slope` and `duration` use it for\nseconds. The spectral features use it for the sample rate.",
"type": [
"string",
"null"
],
"x-message-field": true
}
},
"required": [
"field"
],
"title": "features",
"type": "object"
},
"FileFormat": {
"description": "The layout of the messages in a file.\n\nBoth formats are JSON. Use `ndjson` for a stream. An `ndjson` file is valid\nafter each batch, so you can read it while the pipeline runs or after a\ncrash.",
"oneOf": [
{
"const": "ndjson",
"description": "One JSON message on each line. The output adds each message when it\narrives.",
"type": "string"
},
{
"const": "json_array",
"description": "The file is one JSON array. The output closes the array when the file\nrotates.",
"type": "string"
}
]
},
"FileOutputConfig": {
"description": "Writes each batch to files in a directory on the server.\n\nA `file` connection gives the root directory, and `path` is relative to it.\nThe root must be inside the `--data-dir` of the server. Without that flag,\na `file` output fails to build. The output names each file\n`<open time>-<sequence>.<ext>`, so the names sort by time and are unique.\n\nUse the `file` output for local development and tests.",
"properties": {
"connection": {
"description": "The name of the file connection to write under. Declare the connection\nin the connections file. The connection gives the root directory.",
"type": "string",
"x-connection": "file"
},
"format": {
"anyOf": [
{
"$ref": "#/$defs/FileFormat"
},
{
"type": "null"
}
],
"description": "The layout of the messages. The default is `ndjson`."
},
"path": {
"description": "The directory to write into, relative to the root of the connection,\nfor example `orders`. The path must stay inside the root. kayak refuses\nan absolute path and a path that contains `..`.",
"type": "string"
},
"rotate": {
"anyOf": [
{
"$ref": "#/$defs/RotationConfig"
},
{
"type": "null"
}
],
"description": "When to close a file and start the next file. Without this setting, the\noutput writes one file while the pipeline runs."
}
},
"required": [
"connection",
"path"
],
"title": "file",
"type": "object"
},
"FilterTransformConfig": {
"description": "Keeps the messages that match all `conditions` and drops the other\nmessages. With `invert`, it drops the messages that match and keeps the\nother messages.\n\nThe transform drops a batch that has no messages left. A message that does\nnot have the field of a condition does not match that condition. A message\nwith a field of the wrong type also does not match. Thus `invert` keeps\nthese messages.",
"properties": {
"conditions": {
"description": "The conditions that a message must match. At least one condition is\nrequired.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"invert": {
"description": "Drop the messages that match, and keep the other messages. The default\nis false.",
"type": "boolean"
}
},
"required": [
"conditions"
],
"title": "filter",
"type": "object"
},
"HttpAuthConfig": {
"description": "A credential in a header. The `http` input checks it on each post. The\n`http` output, the `http` transform and the `http_poll` input send it on\neach request.\n\nThis credential is for one pipeline only. It is not related to the user\naccounts of the server.\n\nThe sender sends the same token on each request. Thus the token is only as\nsecure as the connection. kayak serves plain HTTP. Put TLS in front of\nkayak, or other systems on the network path can read the token.",
"oneOf": [
{
"description": "A token in the standard `Authorization` header, as\n`Authorization: Bearer TOKEN`. Use this variant if the other system can\nuse that header.",
"properties": {
"token": {
"$ref": "#/$defs/Secret",
"description": "The token. Use a `${NAME}` reference, so that the config file keeps\nonly the name and the secret store keeps the value."
},
"type": {
"const": "bearer",
"type": "string"
}
},
"required": [
"type",
"token"
],
"type": "object"
},
{
"description": "A fixed value in a header that you name. Use this variant for a system\nthat cannot use the `Authorization` header.",
"properties": {
"name": {
"description": "The name of the header. The `http` input compares the name without\ncase. On an `http` input, the name must not be a header that an\n`envelope` copies into the messages.",
"type": "string"
},
"type": {
"const": "header",
"type": "string"
},
"value": {
"$ref": "#/$defs/Secret",
"description": "The exact value of the header. Use a `${NAME}` reference."
}
},
"required": [
"type",
"name",
"value"
],
"type": "object"
}
]
},
"HttpBodyKind": {
"description": "The body of one request from an `http` output or an `http` transform.\n\nSelect the value that the API at the endpoint expects. Use `batch` for an\nendpoint that takes an array. Use `message` for a webhook that takes one\nevent for each call.",
"oneOf": [
{
"const": "batch",
"description": "The full batch as one JSON array, in one request. The default.",
"type": "string"
},
{
"const": "message",
"description": "One request for each message. The body is the message. The requests go\nin sequence. The first failure fails the batch, and kayak does not send\nthe messages after it.",
"type": "string"
}
]
},
"HttpInputConfig": {
"description": "Accepts messages that are posted to the endpoint of the pipeline,\n`POST /api/pipelines/{id}/messages`.\n\nkayak makes the endpoint from the pipeline id. The endpoint is available\nwhen the pipeline runs. The body is one JSON message or an array of\nmessages. An array becomes one batch. A pipeline can have only one `http`\ninput. A second `http` input fails to build.",
"properties": {
"auth": {
"anyOf": [
{
"$ref": "#/$defs/HttpAuthConfig"
},
{
"type": "null"
}
],
"description": "The credential that a post must have. If you do not set it, the\nendpoint accepts all posts. A post without the correct credential gets\n`401`."
},
"capacity": {
"description": "The maximum number of posted batches in the queue before the pipeline.\nThe default is 1024. When the queue is full, the endpoint refuses a\npost with `503`.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
}
},
"title": "http",
"type": "object"
},
"HttpOutputConfig": {
"description": "Sends the batch to an http endpoint, for example a webhook or an ingest\nAPI.\n\nThe output ignores the body of the reply. A status other than 2xx fails the\nbatch. The error contains the reply of the endpoint. Use the `http`\ntransform if the pipeline needs the reply.",
"properties": {
"auth": {
"anyOf": [
{
"$ref": "#/$defs/HttpAuthConfig"
},
{
"type": "null"
}
],
"description": "The credential that the output sends. If you do not set it, the output\nsends no credential."
},
"body": {
"anyOf": [
{
"$ref": "#/$defs/HttpBodyKind"
},
{
"type": "null"
}
],
"description": "The content of one request. The default is `batch`, one request for\neach batch."
},
"timeout_seconds": {
"description": "The maximum time for one request, in s. The default is 30. A request\nthat times out fails the batch. Thus a slow endpoint stops the pipeline\nfor this time at most.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"url": {
"description": "The endpoint to send to, for example\n`https://example.com/hooks/readings`.",
"type": "string"
},
"verb": {
"anyOf": [
{
"$ref": "#/$defs/HttpVerb"
},
{
"type": "null"
}
],
"description": "The http method. The default is `POST`. `GET` and `DELETE` fail to\nbuild, because a request with no body cannot send the messages."
}
},
"required": [
"url"
],
"title": "http",
"type": "object"
},
"HttpPollConfig": {
"description": "Gets a url at an interval and sends the full reply each time.\n\nEach read is a `GET`. A reply that is an array gives one message for each\nelement. Any other reply gives one message. Use `items` for a reply that\nhas the records inside it, for example `{\"data\": {\"machines\": [...]}}`.\n\nUse this input for reference data, for example a list of machines or\nrecipes that changes rarely. The input has no watermark and no pages. Each\nread sends all records again. Send them to an output that writes the latest\nvalue for each key.\n\nA read fails when the url cannot be reached, when the status is not 2xx,\nwhen the body is not JSON, or when `items` finds nothing. The input reports\nthe failure one time and tries again with backoff. The interval starts\nagain after the next successful read.",
"properties": {
"auth": {
"anyOf": [
{
"$ref": "#/$defs/HttpAuthConfig"
},
{
"type": "null"
}
],
"description": "The credential that the input sends. If you do not set it, the input\nsends no credential."
},
"interval_secs": {
"description": "The time between two reads, in s. The time starts at the end of one\nread. The first read occurs when the pipeline starts.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"items": {
"description": "The position of the records in the reply, as a JSON pointer. For\nexample, `/data/machines` reads the array at `data.machines`. If you do\nnot set it, the input uses the full reply. An array gives one message\nfor each element. Any other value gives one message.",
"type": [
"string",
"null"
]
},
"max_batch": {
"description": "The maximum number of messages in one batch. The default is 1. The input\nputs only messages that are already read into a batch. It does not wait\nfor a batch to fill.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"timeout_seconds": {
"description": "The maximum time for one request, in s. The default is 30.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"url": {
"description": "The url to get, for example `https://erp.example.com/api/machines`.",
"type": "string"
}
},
"required": [
"url",
"interval_secs"
],
"title": "http_poll",
"type": "object"
},
"HttpResponseKind": {
"description": "What an `http` transform does with the reply.",
"oneOf": [
{
"const": "replace",
"description": "The reply becomes the new batch. With `body: batch`, the reply must be a\nJSON array of messages. With `body: message`, the reply is a message or\nan array of messages. The default.",
"type": "string"
},
{
"const": "merge",
"description": "The transform writes the reply onto the message that caused it, under\n`as`. With `body: batch`, an array reply with one entry for each message\ngoes to the messages in sequence. Any other reply goes onto each\nmessage. The messages keep all their fields.",
"type": "string"
}
]
},
"HttpTransformConfig": {
"description": "Sends the batch to an http endpoint and continues with the reply. Use it\nto call a model or another service.\n\n`body` sets the content of one request: the full batch as a JSON array, or\none message. `wrap` puts the body under a key, for example\n`{\"instances\": …}`. `response` sets what the transform does with the reply.\n`replace` makes the reply the new batch. `merge` writes the reply onto the\nmessage under `as`. `unwrap` reads the reply from under a key first.\n\nA status other than 2xx fails the batch. The error contains the reply of the\nendpoint. A network failure, a 5xx or a 429 is tried again `retries` times\nwith backoff.\n\nFor a model, put a `buffer` and a `features` transform before this\ntransform. Use `response: merge` to keep the identifiers of the message.",
"properties": {
"as": {
"description": "The field to write the reply under. Applies to `response: merge`.",
"type": [
"string",
"null"
]
},
"auth": {
"anyOf": [
{
"$ref": "#/$defs/HttpAuthConfig"
},
{
"type": "null"
}
],
"description": "The credential that the transform sends. If you do not set it, the\ntransform sends no credential."
},
"body": {
"anyOf": [
{
"$ref": "#/$defs/HttpBodyKind"
},
{
"type": "null"
}
],
"description": "The content of one request. The default is `batch`."
},
"response": {
"anyOf": [
{
"$ref": "#/$defs/HttpResponseKind"
},
{
"type": "null"
}
],
"description": "What to do with the reply. The default is `replace`."
},
"retries": {
"description": "The number of times to send a request again before the batch fails.\nApplies to a network failure, a 5xx and a 429. The default is 0. Each\ntry waits longer than the previous try, and the pipeline waits too.",
"format": "uint32",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"timeout_seconds": {
"description": "The maximum time for one request, in s. The default is 30.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"unwrap": {
"description": "A key to read the reply from, for an API that replies with\n`{\"predictions\": …}`.",
"type": [
"string",
"null"
]
},
"url": {
"description": "The endpoint to send to.",
"type": "string"
},
"verb": {
"$ref": "#/$defs/HttpVerb",
"description": "The http method. `GET` and `DELETE` fail to build, because a request\nwith no body cannot send the messages."
},
"wrap": {
"description": "A key to put the body under, for an API that expects `{\"key\": …}`.",
"type": [
"string",
"null"
]
}
},
"required": [
"url",
"verb"
],
"title": "http",
"type": "object"
},
"HttpVerb": {
"description": "The http method of a request.",
"enum": [
"GET",
"POST",
"PUT",
"PATCH",
"DELETE"
],
"type": "string"
},
"InduInputConfig": {
"description": "Reads live sensors and streams from Indu Cloud through `/api/v1/live/sse`,\nwith the API key of the connection.\n\nName sensors and streams with the ids that the platform uses. Do not use\nUUIDs. The input finds the names through `/api/v1` on the first read. If\nthe key cannot find or see a name, the input reports an error. It then\ntries again after a pause.\n\nEach reading is one message, for example\n`{\"kind\": \"sensor\", \"name\": \"press-3/temperature\", \"value\": 71.2, \"at\": …}`.\nThe message also contains the ids of the platform. When the connection\ndrops, the input connects again with backoff. If the input cannot read\nall readings, it reports an error.",
"properties": {
"backfill": {
"description": "Send the latest value of each series before the live readings. The\ndefault is true. Thus a restarted pipeline has a value for each series\nimmediately.",
"type": [
"boolean",
"null"
]
},
"connection": {
"description": "The name of the indu connection to read through. Declare the connection\nin the connections file.",
"type": "string",
"x-connection": "indu"
},
"max_batch": {
"description": "The maximum number of readings in one batch. The default is 1. The input\nputs only readings that are already received into a batch. It does not\nwait for more readings.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"sensors": {
"description": "The sensors to read, as `<device>/<sensor>`, for example\n`press-3/temperature`. Use the ids that the platform uses. kayak divides\nthe name at the first `/`.",
"items": {
"type": "string"
},
"type": "array"
},
"streams": {
"description": "The streams to read, by the name they were written under, for example\n`press-3/oee`. For a stream that the platform calculates, use its\ndisplay name.",
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"connection"
],
"title": "indu",
"type": "object"
},
"InduOutputConfig": {
"description": "Writes messages into Indu Cloud as streams through\n`POST /ingest/v1/streams`. A stream is a series that is not a sensor.\n\nEach message gives one reading for each entry in `series`. For example, a\nreducer that sends `{machine, oee, availability}` with two series entries\nwrites two streams for each machine. Indu creates an unknown stream when\nthe key of the connection has permission to create streams. If Indu does\nnot accept all rows, the batch fails. The error contains the row errors\nfrom Indu.",
"properties": {
"at": {
"description": "The field that contains the time of the reading, as an RFC 3339 string\nor as ms since the epoch. If you do not set it, the output uses the time\nat which it sends the batch. An `envelope` puts the receive time of an\ninput at `_meta.received_at`.",
"type": [
"string",
"null"
]
},
"connection": {
"description": "The name of the indu connection to write through. Declare the\nconnection in the connections file.",
"type": "string",
"x-connection": "indu"
},
"series": {
"description": "The streams to write, one reading for each message. At least one entry\nis required.",
"items": {
"$ref": "#/$defs/InduSeries"
},
"type": "array"
},
"timeout_seconds": {
"description": "The maximum time for one request, in s. The default is 30.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
}
},
"required": [
"connection",
"series"
],
"title": "indu",
"type": "object"
},
"InduSeries": {
"description": "One series that an `indu` output writes: the stream that gets the value,\nand the field that contains the value.",
"properties": {
"stream": {
"description": "The name of the stream in Indu, for example `press-3/oee`. The name can\ncontain `{field}` placeholders that the output fills from the message,\nfor example `{machine}/oee`. Thus one output can write a stream for each\nmachine. The output skips this series for a message that does not have\nthe field of a placeholder.",
"type": "string"
},
"unit": {
"description": "The unit that Indu records when it creates the stream, for example `%`.\nIndu ignores it when the stream exists.",
"type": [
"string",
"null"
]
},
"value": {
"description": "The field that contains the value, as a path (`oee`, `stats.mean`). The\nvalue must be a number. The output skips this series for a message\nwhere the value is missing or is not a number. The batch does not fail.",
"type": "string"
}
},
"required": [
"stream",
"value"
],
"type": "object"
},
"InputConfig": {
"oneOf": [
{
"$ref": "#/$defs/DummyConfig",
"properties": {
"type": {
"const": "dummy",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/HttpInputConfig",
"properties": {
"type": {
"const": "http",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/KafkaConfig",
"properties": {
"type": {
"const": "kafka",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/NatsConfig",
"properties": {
"type": {
"const": "nats",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/PipelineConfig",
"properties": {
"type": {
"const": "pipeline",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/MqttConfig",
"properties": {
"type": {
"const": "mqtt",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/RedisConfig",
"properties": {
"type": {
"const": "redis",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/OpcuaConfig",
"properties": {
"type": {
"const": "opcua",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/InduInputConfig",
"properties": {
"type": {
"const": "indu",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/PostgresInputConfig",
"properties": {
"type": {
"const": "postgres",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/ClickhouseInputConfig",
"properties": {
"type": {
"const": "clickhouse",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/HttpPollConfig",
"properties": {
"type": {
"const": "http_poll",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
}
],
"properties": {
"ack": {
"anyOf": [
{
"$ref": "#/$defs/AckMode"
},
{
"type": "null"
}
],
"description": "When the input acknowledges a message to its broker. The default is\n`on_receipt`. Only the `kafka` and `mqtt` inputs support `on_delivery`.\nThe `mqtt` input requires a `qos` of `at_least_once` or higher for it.\nOn all other inputs, `on_delivery` fails to build."
},
"buffer": {
"anyOf": [
{
"$ref": "#/$defs/BufferConfig"
},
{
"type": "null"
}
],
"description": "Collect messages from this input into batches before the transforms.\nUse a count (`static`), a time (`tumbling`) or the first of the two\n(`batch`). The buffer never sends an empty batch. Available on all\ninput types. This is not the `buffer` transform."
},
"envelope": {
"anyOf": [
{
"$ref": "#/$defs/EnvelopeConfig"
},
{
"type": "null"
}
],
"description": "Add metadata about the source of each message, for example the subject,\nthe topic or the partition. The \"metadata\" section lists the fields.\nAvailable on all input types. If you do not set it, the input sends each\nmessage without changes."
}
},
"type": "object"
},
"KafkaConfig": {
"description": "Consumes JSON messages from a kafka topic.\n\nThe input skips a payload that is not JSON and writes a warning to the log.\nThe consumer connects on the first read and joins a consumer group. Kafka\nkeeps the read position of the group between restarts.",
"properties": {
"connection": {
"description": "The name of the kafka connection to consume from. Declare the\nconnection in the connections file.",
"type": "string",
"x-connection": "kafka"
},
"group": {
"description": "The consumer group id. Kafka keeps one read position for each group.\nTwo pipelines in the same group divide the topic between them. Two\npipelines in different groups each get all messages.",
"type": "string"
},
"max_batch": {
"description": "The maximum number of messages in one batch. The default is 1.\n\nThe input puts only records that are already received into a batch. It\ndoes not wait for more records, so a high value does not add latency on\na quiet topic. Increase it when the consumer reads a backlog. Each batch\nhas a fixed cost in the pipeline and in each downstream pipeline.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"start_at": {
"anyOf": [
{
"$ref": "#/$defs/KafkaStartAt"
},
{
"type": "null"
}
],
"description": "The start position when the group has no committed position. `earliest`\nreads the topic from the start. `latest` reads only new messages. The\ndefault is `latest`."
},
"topic": {
"description": "The topic to consume from.",
"type": "string"
}
},
"required": [
"connection",
"topic",
"group"
],
"title": "kafka",
"type": "object"
},
"KafkaOutputConfig": {
"description": "Publishes each message in the batch to a kafka topic, one record for each\nmessage.\n\nThe records have no key, so kafka distributes them across the partitions\nof the topic.",
"properties": {
"connection": {
"description": "The name of the kafka connection to publish to. Declare the connection\nin the connections file.",
"type": "string",
"x-connection": "kafka"
},
"topic": {
"description": "The topic to publish to.",
"type": "string"
}
},
"required": [
"connection",
"topic"
],
"title": "kafka",
"type": "object"
},
"KafkaStartAt": {
"description": "The position at which a new consumer group starts to read.",
"enum": [
"earliest",
"latest"
],
"type": "string"
},
"KeepPolicy": {
"description": "Which fields of the input message a `map` keeps.",
"oneOf": [
{
"const": "all",
"description": "Keep all fields, and apply the mappings to them. This is the default.",
"type": "string"
},
{
"const": "mapped",
"description": "Keep only the fields that the mappings wrote. Use it to prepare a\nmessage for an output with a fixed shape, for example a `postgres`\ntable. It also removes intermediate fields. You cannot use `drop` with\n`mapped`.",
"type": "string"
}
]
},
"Literal": {
"description": "A literal value. A `constant` writes it, and a `default` writes it in place\nof a missing field. The `type` field selects the type of the value.",
"oneOf": [
{
"description": "A string.",
"properties": {
"type": {
"const": "text",
"type": "string"
},
"value": {
"description": "The text.",
"type": "string"
}
},
"required": [
"type",
"value"
],
"type": "object"
},
{
"description": "A number.",
"properties": {
"type": {
"const": "number",
"type": "string"
},
"value": {
"description": "The number.",
"format": "double",
"type": "number"
}
},
"required": [
"type",
"value"
],
"type": "object"
},
{
"description": "True or false.",
"properties": {
"type": {
"const": "boolean",
"type": "string"
},
"value": {
"description": "The boolean value.",
"type": "boolean"
}
},
"required": [
"type",
"value"
],
"type": "object"
},
{
"description": "JSON `null`. The field is present with the value `null`.",
"properties": {
"type": {
"const": "null",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
}
]
},
"MapMissingPolicy": {
"description": "What `map` does with a message that does not have a field that a mapping\nreads.\n\nNo value drops the message. To drop a message, use `filter`.",
"oneOf": [
{
"const": "error",
"description": "Fail the batch. This is the default. To accept a missing field, use\n`omit`, or give the mapping a `default`.",
"type": "string"
},
{
"const": "omit",
"description": "Do not write the target field.",
"type": "string"
},
{
"const": "null",
"description": "Write the target field as `null`.",
"type": "string"
}
]
},
"MapTransformConfig": {
"description": "Changes the shape of every message. A mapping can rename, move, cast or\nremove a field, or write a constant. The transform applies the mappings in\norder.\n\nEach entry in `mappings` reads fields from the message and writes one\nfield. A mapping can read the fields that earlier mappings wrote. Use this\nfor intermediate values. With `keep: all`, a `drop` can remove them again.\n\nReads and writes use dotted paths. For example, an `as` of `sensor.id`\nwrites the value inside a `sensor` object. If the object does not exist,\nthe transform makes it.\n\nBy default, the message passes through with the mappings applied to it.\n`keep` can change this. One message goes in and one message comes out. To\ndrop a message, use `filter`. To make many messages, use `splitter`.",
"properties": {
"keep": {
"$ref": "#/$defs/KeepPolicy",
"description": "Which fields of the input message the output keeps. The default is\n`all`."
},
"mappings": {
"description": "The fields to write, in order. Give one mapping or more. Two mappings\nmust not write the same field.",
"items": {
"$ref": "#/$defs/Mapping"
},
"type": "array"
},
"on_missing": {
"$ref": "#/$defs/MapMissingPolicy",
"description": "What to do with a message that does not have a field that a mapping\nreads. The default is `error`. A `default` on the mapping applies\nfirst. Use a `default` when you expect one field to be absent."
}
},
"required": [
"mappings"
],
"title": "map",
"type": "object"
},
"Mapping": {
"description": "One field that the transform writes onto the message, and the source of\nits value. The `type` field selects the mapping.",
"oneOf": [
{
"description": "Reads a value from one field and writes it to another field. Use it to\nrename a field, or to move a field out of a nested object, for example\nfrom `_meta.subject` to `subject`.",
"properties": {
"as": {
"description": "The field to write. The default is the last segment of `from`.",
"type": [
"string",
"null"
]
},
"default": {
"anyOf": [
{
"$ref": "#/$defs/Literal"
},
{
"type": "null"
}
],
"description": "The value to write when the message does not have `from`. With it,\n`on_missing` does not apply."
},
"from": {
"description": "The field to read, as a dotted path.",
"type": "string"
},
"type": {
"const": "copy",
"type": "string"
}
},
"required": [
"type",
"from"
],
"type": "object"
},
{
"description": "Writes a fixed value, for example the name of the site.",
"properties": {
"as": {
"description": "The field to write the value to.",
"type": "string"
},
"type": {
"const": "constant",
"type": "string"
},
"value": {
"$ref": "#/$defs/Literal",
"description": "The value to write."
}
},
"required": [
"type",
"value",
"as"
],
"type": "object"
},
{
"description": "Writes the value of the first field in a list that the message has.\nUse it when two sources use different names for the same field.",
"properties": {
"as": {
"description": "The field to write the first value to.",
"type": "string"
},
"default": {
"anyOf": [
{
"$ref": "#/$defs/Literal"
},
{
"type": "null"
}
],
"description": "The value to write when the message has none of the fields."
},
"from": {
"description": "The fields to try, in order. Give two fields or more.",
"items": {
"type": "string"
},
"type": "array"
},
"type": {
"const": "coalesce",
"type": "string"
}
},
"required": [
"type",
"from",
"as"
],
"type": "object"
},
{
"description": "Converts a value from one JSON type to another. For example, it\nconverts the string `\"12.5\"` to the number `12.5`, or epoch seconds to\na timestamp. It can also parse a string that contains JSON.\n\nThis is the only place in kayak that converts a value. The column\nmapping of the database outputs checks a value and does not convert it.",
"properties": {
"as": {
"description": "The field to write. The default is the last segment of `from`. For\nexample, `{\"from\": \"value\", \"to\": \"float\"}` converts `value` in place.",
"type": [
"string",
"null"
]
},
"default": {
"anyOf": [
{
"$ref": "#/$defs/Literal"
},
{
"type": "null"
}
],
"description": "The value to write when the message does not have `from`. A value\nthat is present and does not convert is always an error."
},
"from": {
"description": "The field to read.",
"type": "string"
},
"to": {
"$ref": "#/$defs/CastType",
"description": "The type to convert the value to."
},
"type": {
"const": "cast",
"type": "string"
}
},
"required": [
"type",
"from",
"to"
],
"type": "object"
},
{
"description": "Joins fields and literal text into one string. For example, use it to\nwrite a `site/machine` key for `group_by`.",
"properties": {
"as": {
"description": "The field to write the string to.",
"type": "string"
},
"parts": {
"description": "The parts, in order. Give one part or more.",
"items": {
"$ref": "#/$defs/ConcatPart"
},
"type": "array"
},
"type": {
"const": "concat",
"type": "string"
}
},
"required": [
"type",
"parts",
"as"
],
"type": "object"
},
{
"description": "One arithmetic operation on two numbers. Each number is a field or a\nliteral.\n\nFor a calculation with more steps, use more mappings with intermediate\nfields. For example, `(f - 32) / 1.8` is two mappings. For a long\ncalculation, use the `script` transform.",
"properties": {
"as": {
"description": "The field to write the result to.",
"type": "string"
},
"left": {
"$ref": "#/$defs/Operand",
"description": "The left operand."
},
"on_zero": {
"$ref": "#/$defs/OnZero",
"description": "For `divide`: what to do when the right field holds zero. The\ndefault is `error`, which fails the batch."
},
"operator": {
"$ref": "#/$defs/ArithmeticOperator",
"description": "The operation to do."
},
"right": {
"$ref": "#/$defs/Operand",
"description": "The right operand."
},
"type": {
"const": "arithmetic",
"type": "string"
}
},
"required": [
"type",
"left",
"operator",
"right",
"as"
],
"type": "object"
},
{
"description": "Writes the start of the calendar period that a time is in, for example\nthe hour, the day or the shift.\n\nUse the field in `group_by`. A stateful transform that groups by it\nkeeps one series for each period. The idle timeout of the state bucket\nremoves the old periods.\n\nThe periods use the local clock of `timezone`. Thus, a shift that\nstarts at 06:00 starts at 06:00 in summer and in winter. On the night\nthat the clock changes, the shift is 7 or 9 hours.\n\nThe periods start at local midnight on 1 January 1970, plus\n`offset_seconds`. For example, `every_seconds: 28800` with\n`offset_seconds: 21600` gives 06:00, 14:00 and 22:00. A week starts on a\nThursday. For a week that starts on a Monday, add an offset of four days.",
"properties": {
"as": {
"description": "The field to write the start time to.",
"type": "string"
},
"every_seconds": {
"description": "The length of a period, in seconds.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"format": {
"anyOf": [
{
"$ref": "#/$defs/TimeFormat"
},
{
"type": "null"
}
],
"description": "The format of the start time. The default is `rfc3339`."
},
"from": {
"description": "The field that holds the time, as an RFC 3339 string or as\nmilliseconds since the epoch.",
"type": "string"
},
"offset_seconds": {
"description": "The offset of the start of the periods, in seconds. It must be\nless than `every_seconds`.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"timezone": {
"description": "The IANA time zone of the periods, for example `Europe/Stockholm`.\nThe default is UTC.",
"type": [
"string",
"null"
]
},
"type": {
"const": "time_bucket",
"type": "string"
}
},
"required": [
"type",
"from",
"every_seconds",
"as"
],
"type": "object"
},
{
"description": "Removes fields from the message. Use it to remove metadata fields\nbefore the output. A field that is not there is not an error.\n`on_missing` does not apply.",
"properties": {
"from": {
"description": "The fields to remove. Give one field or more.",
"items": {
"type": "string"
},
"type": "array"
},
"type": {
"const": "drop",
"type": "string"
}
},
"required": [
"type",
"from"
],
"type": "object"
}
]
},
"MissingColumnPolicy": {
"description": "What to do with a message that does not have the field of a column. A\nfield with the value `null` counts as missing.",
"oneOf": [
{
"const": "null",
"description": "Write `NULL`. This is the default for a nullable column.",
"type": "string"
},
{
"const": "error",
"description": "Fail the batch. This is the default for a column with\n`\"nullable\": false`.",
"type": "string"
},
{
"const": "skip_row",
"description": "Do not write the message. The output writes no column of that row.",
"type": "string"
}
]
},
"MissingFieldPolicy": {
"description": "What to do with a message that does not have a field that is aggregated or\ngrouped by. A field with the value `null` is missing.",
"oneOf": [
{
"const": "error",
"description": "Fail the batch. The default. A sum of only some of the messages gives a\nwrong result that is not visible downstream.",
"type": "string"
},
{
"const": "skip",
"description": "Do not use that message in that aggregation. An aggregation with no\nvalues gives `null`, or `0` for the counts.",
"type": "string"
}
]
},
"MqttConfig": {
"description": "Subscribes to an mqtt topic and parses each message as JSON.\n\nThe topic can be a filter with the mqtt wildcards `+` and `#`. The input\nskips a payload that is not JSON and writes a warning to the log.\n\nThe input opens the connection on the first read. kayak makes the client id\nfrom the pipeline id and the topic. You cannot set the client id.",
"properties": {
"connection": {
"description": "The name of the mqtt connection to subscribe on. Declare the connection\nin the connections file.",
"type": "string",
"x-connection": "mqtt"
},
"max_batch": {
"description": "The maximum number of messages in one batch. The default is 1.\n\nThe input puts only messages that are already received into a batch.\nIt does not wait for more messages, so a high value does not add\nlatency on a quiet topic.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"qos": {
"anyOf": [
{
"$ref": "#/$defs/MqttQos"
},
{
"type": "null"
}
],
"description": "The quality of service for the subscription. The default is\n`at_most_once`. `ack: on_delivery` requires `at_least_once` or\n`exactly_once`."
},
"topic": {
"description": "The topic or topic filter to subscribe to.",
"type": "string"
}
},
"required": [
"connection",
"topic"
],
"title": "mqtt",
"type": "object"
},
"MqttOutputConfig": {
"description": "Publishes each message in the batch to an mqtt topic, one message for each\npublish.\n\nkayak makes the client id from the pipeline id and the topic. You cannot\nset the client id.",
"properties": {
"connection": {
"description": "The name of the mqtt connection to publish on. Declare the connection\nin the connections file.",
"type": "string",
"x-connection": "mqtt"
},
"qos": {
"anyOf": [
{
"$ref": "#/$defs/MqttQos"
},
{
"type": "null"
}
],
"description": "The quality of service for each publish. The default is `at_most_once`."
},
"retain": {
"description": "Tell the broker to keep the message as the retained message of the\ntopic. The broker sends it to each new subscriber. The default is false.",
"type": [
"boolean",
"null"
]
},
"topic": {
"description": "The topic to publish to.",
"type": "string"
}
},
"required": [
"connection",
"topic"
],
"title": "mqtt",
"type": "object"
},
"MqttQos": {
"description": "The mqtt quality of service for a subscribe or a publish.",
"oneOf": [
{
"const": "at_most_once",
"description": "QoS 0. The broker does not send a message again and there is no\nacknowledgement. The default.",
"type": "string"
},
{
"const": "at_least_once",
"description": "QoS 1. The broker sends a message again until it gets an\nacknowledgement, so a message can arrive more than one time. An input\nwith `ack: on_delivery` requires this level or higher.",
"type": "string"
},
{
"const": "exactly_once",
"description": "QoS 2. A four-part handshake makes sure of exactly one delivery. This\nlevel has the highest cost. Use `at_least_once` if a duplicate message\nis not a problem.",
"type": "string"
}
]
},
"NatsConfig": {
"description": "Subscribes to a nats subject and parses each message as JSON.\n\nThe input skips a payload that is not JSON and writes a warning to the log.\nThe input opens the connection on the first read.",
"properties": {
"connection": {
"description": "The name of the nats connection to subscribe on. Declare the connection\nin the connections file.",
"type": "string",
"x-connection": "nats"
},
"max_batch": {
"description": "The maximum number of messages in one batch. The default is 1.\n\nThe input puts only messages that are already received into a batch.\nIt does not wait for more messages, so a high value does not add\nlatency on a quiet subject.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"subject": {
"description": "The subject to subscribe to.",
"type": "string"
}
},
"required": [
"connection",
"subject"
],
"title": "nats",
"type": "object"
},
"NatsOutputConfig": {
"description": "Publishes each message in the batch to a nats subject, one message for each\npublish.",
"properties": {
"connection": {
"description": "The name of the nats connection to publish on. Declare the connection\nin the connections file.",
"type": "string",
"x-connection": "nats"
},
"subject": {
"description": "The subject to publish to.",
"type": "string"
}
},
"required": [
"connection",
"subject"
],
"title": "nats",
"type": "object"
},
"NumericFilterOperatorKind": {
"description": "How a `numeric` condition compares a number to the value in the config.",
"enum": [
"greater_than",
"less_than",
"equal_to",
"not_equal_to"
],
"type": "string"
},
"OnZero": {
"description": "What a `divide` does when the right field holds zero. For example, this\noccurs with a ratio over an empty period.",
"oneOf": [
{
"description": "Fail the batch. The error names the division. This is the default.",
"properties": {
"type": {
"const": "error",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"description": "Write `null` as the result.",
"properties": {
"type": {
"const": "null",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"description": "Write a number as the result.",
"properties": {
"type": {
"const": "value",
"type": "string"
},
"value": {
"description": "The number to write.",
"format": "double",
"type": "number"
}
},
"required": [
"type",
"value"
],
"type": "object"
}
]
},
"OpcuaBrowseConfig": {
"description": "All variables under a node in the address space of the server. The input\nbrowses for them when the pipeline starts.\n\nThe input reads the address space only at the start. A tag that is added\nto the server later is read only after a restart. A tag that is removed\nstops without an error. Use a `nodes` list to name in the config file\nexactly which nodes the input reads. You can use `browse` and `nodes`\ntogether.",
"properties": {
"depth": {
"description": "The number of levels below the root to follow. The default is 3. The\nvalue 0 is not permitted. There is no value for \"all levels\", because\nthe address space of a plant server can have thousands of nodes.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"root": {
"description": "The id of the node to browse under, in the same notation as `node_id`.\nThis is usually a folder, for example `ns=2;s=Machine1`. The input\nsubscribes to each variable under it. It follows folders and objects\nbut does not subscribe to them.",
"type": "string"
}
},
"required": [
"root"
],
"title": "opcua browse",
"type": "object"
},
"OpcuaConfig": {
"description": "Subscribes to variables on an OPC UA server and sends one message for each\nchange of a value.\n\nThe input makes a subscription with one monitored item for each node. The\nserver sends a value when it changes. The input does not poll. A tag that\ndoes not change sends no messages. `publish_interval_ms` sets how\nfrequently the server can send. It does not set how frequently the server\nsamples.\n\nEach message is one reading. It contains the tag and the value:\n\n```json\n{\n \"node\": \"ns=2;s=Machine1.Temperature\",\n \"name\": \"temperature\",\n \"value\": 21.5,\n \"status\": \"Good\",\n \"source_timestamp\": \"2026-01-01T12:00:00.123Z\",\n \"server_timestamp\": \"2026-01-01T12:00:00.130Z\"\n}\n```\n\n`status` is the quality of the reading and is always present. A failed\nsensor sends a `Bad...` status with a `null` value. Use a `filter` to remove\nthese readings. `source_timestamp` is the time at which the device produced\nthe value. Use it to reduce or partition. The `received_at` field of the\nenvelope is the time at which kayak read the value.\n\nSet `nodes`, `browse` or both. One of them is required. The input subscribes\nto a node only one time, also when two settings name it.",
"properties": {
"browse": {
"anyOf": [
{
"$ref": "#/$defs/OpcuaBrowseConfig"
},
{
"type": "null"
}
],
"description": "A node to browse. The input subscribes to each variable under it."
},
"connection": {
"description": "The name of the opcua connection to subscribe on. Declare the\nconnection in the connections file.",
"type": "string",
"x-connection": "opcua"
},
"deadband": {
"description": "The smallest change of a value that the server reports, in the units of\nthe value. If you do not set it, the server reports each change.\n\nThe server applies the deadband, so it decreases network traffic. It\napplies only to numeric nodes. The server reports each change of a\nstring or a boolean.",
"format": "double",
"type": [
"number",
"null"
]
},
"max_batch": {
"description": "The maximum number of messages in one batch. The default is 1.\n\nOne publish from the server contains each node that changed in the\ninterval. With 200 tags at 1 Hz and the default, the pipeline handles\n200 batches each second. A higher value decreases this cost. The input\nputs only changes that are already received into a batch.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"nodes": {
"description": "The nodes to subscribe to, one entry for each node.",
"items": {
"$ref": "#/$defs/OpcuaNodeConfig"
},
"type": "array"
},
"publish_interval_ms": {
"description": "The interval at which the server can send a group of changes, in ms.\nThe default is 1000. This value sets the longest time that a change\nwaits.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"queue_size": {
"description": "The number of samples that the server keeps for one node between two\npublishes. The default is 1. With 1, the server sends only the latest\nvalue of a node that changes two times in one interval. Increase it,\ntogether with `sampling_interval_ms`, when you need each sample.",
"format": "uint32",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"sampling_interval_ms": {
"description": "The interval at which the server samples each node, in ms. If you do not\nset it, the server samples at the publish interval. Set a smaller value\nto get more readings between two publishes.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
}
},
"required": [
"connection"
],
"title": "opcua",
"type": "object"
},
"OpcuaNodeConfig": {
"description": "One node that an `opcua` input subscribes to, and the name for it in the\nmessages.",
"properties": {
"name": {
"description": "The name of the node in the messages. The default is the node id. Set\na readable name to use in a `group_by` or a column mapping.",
"type": [
"string",
"null"
]
},
"node_id": {
"description": "The id of the node, in OPC UA notation. Use `ns=2;s=Machine1.Temperature`\nfor a string identifier and `ns=2;i=1042` for a numeric identifier. Use\n`g=` for a GUID and `b=` for an opaque identifier. A node id with no\n`ns=` is in namespace 0, the namespace of the server.",
"type": "string"
}
},
"required": [
"node_id"
],
"title": "opcua node",
"type": "object"
},
"Operand": {
"description": "One side of an [`Mapping::Arithmetic`]: a field to read, or a fixed number.",
"oneOf": [
{
"description": "A number from the message.",
"properties": {
"field": {
"description": "The field to read. It must hold a number.",
"type": "string"
},
"type": {
"const": "field",
"type": "string"
}
},
"required": [
"type",
"field"
],
"type": "object"
},
{
"description": "A number in the config.",
"properties": {
"type": {
"const": "value",
"type": "string"
},
"value": {
"description": "The number.",
"format": "double",
"type": "number"
}
},
"required": [
"type",
"value"
],
"type": "object"
}
]
},
"OutputConfig": {
"oneOf": [
{
"$ref": "#/$defs/StdoutOutputConfig",
"properties": {
"type": {
"const": "stdout",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/FileOutputConfig",
"properties": {
"type": {
"const": "file",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/S3OutputConfig",
"properties": {
"type": {
"const": "s3",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/KafkaOutputConfig",
"properties": {
"type": {
"const": "kafka",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/NatsOutputConfig",
"properties": {
"type": {
"const": "nats",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/PostgresOutputConfig",
"properties": {
"type": {
"const": "postgres",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/ClickhouseOutputConfig",
"properties": {
"type": {
"const": "clickhouse",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/MqttOutputConfig",
"properties": {
"type": {
"const": "mqtt",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/RedisOutputConfig",
"properties": {
"type": {
"const": "redis",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/HttpOutputConfig",
"properties": {
"type": {
"const": "http",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/InduOutputConfig",
"properties": {
"type": {
"const": "indu",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/TidepoolOutputConfig",
"properties": {
"type": {
"const": "tidepool",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
}
],
"type": "object"
},
"PipelineConfig": {
"description": "Reads the output of another pipeline. Use it to connect pipelines into a\ngraph.\n\nMany pipelines can read from the same upstream. Each of them gets all\nbatches. The upstream must exist when kayak creates this pipeline, so\ndeclare the upstream first in the config file.",
"properties": {
"upstream": {
"description": "The id of the pipeline to read from.",
"type": "string",
"x-pipeline-id": true
}
},
"required": [
"upstream"
],
"title": "pipeline",
"type": "object"
},
"PipelineState": {
"description": "The state bucket of a pipeline, and the field that gives the key of each\nmessage.\n\nEach pipeline sets its own key, because the same value can have different\nfield names in two streams. For example, one stream has\n`_meta.machine_id` and another stream has `machine_id`.\n\nMake sure that all pipelines that share a bucket use keys with the same\nvalues. kayak does not check this.",
"properties": {
"bucket": {
"description": "The name of the bucket that this pipeline reads and writes. Declare the\nbucket under `state` at the top of the config. If the bucket is not\ndeclared, the pipeline does not build.",
"type": "string"
},
"key": {
"description": "The field that gives the key, as a dotted path. For example,\n`_meta.machine_id`.\n\nLeave it out for one value for the full bucket. Use that only for an\nitem that has one value. For a value for each device, set `key`.",
"type": [
"string",
"null"
]
}
},
"required": [
"bucket"
],
"title": "pipeline state",
"type": "object"
},
"PivotTransformConfig": {
"description": "Changes a stream with one reading per message into rows. The transform\nkeeps the latest value of each name in `names` for each key. It writes all\nof these values onto every message.\n\nFor example, the input is `{\"sensor\": \"state\", \"value\": \"RUNNING\"}` and\nthen `{\"sensor\": \"fault\", \"value\": \"NONE\"}`. The output row is\n`{\"state\": \"RUNNING\", \"fault\": \"NONE\", ...}`.\n\nWhen the `name` field of a message holds one of `names`, the transform\nfirst records that reading. Then it writes every value that it keeps for\nthe key onto the message. It writes them at the top level, or under\n`into`. A name with no reading yet for the key is not written. One message\ngoes in and one message comes out.\n\n`names` is required because it limits the state. A message with a name that\nis not in `names` adds nothing, but it gets the row.",
"properties": {
"group_by": {
"description": "The fields that identify a row, as in `reduce`. Leave it out for one\nrow.",
"items": {
"type": "string"
},
"type": "array"
},
"into": {
"description": "An object field to write the values under. Leave it out to write them\nat the top level.",
"type": [
"string",
"null"
]
},
"name": {
"description": "The field that holds the name of the reading. kayak compares its value\nwith `names`.",
"type": "string",
"x-message-field": true
},
"names": {
"description": "The names to keep and write. Each name becomes a field. Give one name\nor more.",
"items": {
"type": "string"
},
"type": "array"
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have a group field, or that\nhas a name from `names` and no `value`. The default is `error`."
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"value": {
"description": "The field that holds the reading. The value can be any JSON value, for\nexample a string or a number.",
"type": "string",
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"name",
"value",
"names"
],
"title": "pivot",
"type": "object"
},
"PollMode": {
"description": "Whether a read returns all rows or only the new rows.",
"oneOf": [
{
"description": "Each read returns all rows, in one query. Use it for reference data,\nfor example a table of recipes for a `remember` transform. A snapshot\nhas no page limit. Use it only for a table that fits in memory.",
"properties": {
"type": {
"const": "snapshot",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"description": "Each read returns only the rows with a `field` value above the highest\nvalue that the input already sent. The input reads pages in the order\nof the field. The value of the field must increase, for example an id\nor an `updated_at`. Put an index on the field. Without an index, each\nread scans the full table.",
"properties": {
"field": {
"description": "The column that the input follows. The watermark is the highest\nvalue that the input sent. Each read asks for the rows above the\nwatermark. The input does not read rows where the column is `null`.",
"type": "string"
},
"lag_secs": {
"description": "For a timestamp column: the time to stay behind the current time,\nin seconds. A later read gets the rows that are less than this time\nbefore `now()`. This gives late transactions time to commit. Do not\nuse it with a numeric column. The server refuses the query.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"start_from": {
"anyOf": [
{
"$ref": "#/$defs/StartFrom"
},
{
"type": "null"
}
],
"description": "Where the first read starts. With `newest`, the input reads only\nthe rows added after the pipeline started. With `oldest`, it reads\nall rows first and then follows the table. The default is `newest`."
},
"type": {
"const": "incremental",
"type": "string"
}
},
"required": [
"type",
"field"
],
"type": "object"
}
]
},
"PostgresInputConfig": {
"description": "Reads a postgres table, view or query at an interval and sends each row as\na message.\n\nThe input runs a query each `interval_secs`. In `snapshot` mode, it reads\nthe full relation. In `incremental` mode, it reads only the rows after the\nlast read, by a column that increases. Each row becomes one JSON object\nwith the column names as fields. Postgres makes the object with\n`row_to_json`. A timestamp is ISO 8601, a `numeric` keeps its digits, and\na `jsonb` column is a nested value:\n\n```json\n{\"id\": 42, \"sensor\": \"press-3\", \"value\": 21.5, \"recorded_at\": \"2026-01-01T12:00:00.123456+00:00\"}\n```\n\nAn incremental read is at-least-once across a restart. The input keeps the\nwatermark in memory, and after a restart it starts again from `start_from`.\nAn incremental read does not see a deleted row. Put an index on the column\nthat it follows, or each read scans the full table.",
"properties": {
"columns": {
"description": "The columns to read, in order. Leave it empty to read all columns. For\nan incremental input, the list must include `field`.",
"items": {
"type": "string"
},
"type": "array"
},
"connection": {
"description": "The name of the postgres connection to read through. Declare the\nconnection in the connections file.",
"type": "string",
"x-connection": "postgres"
},
"interval_secs": {
"description": "The time between two reads, in seconds. The time starts at the end of a\nread. Thus, two reads never overlap. The first read occurs when the\npipeline starts.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"max_batch": {
"description": "The maximum number of rows in one batch. The default is 1. The input\nputs rows that it already read into a batch. It does not wait for more\nrows to fill a batch.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"mode": {
"$ref": "#/$defs/PollMode",
"description": "Whether each read returns all rows (`snapshot`) or only the rows after\nthe last read (`incremental`)."
},
"page_size": {
"description": "The maximum number of rows that one query returns. The default is\n1000. When a page is full, the input reads the next page immediately.\nThe interval starts after a page that is not full. A `snapshot` ignores\nthis field and reads all rows in one query.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"query": {
"description": "A `SELECT` to read in place of a table. The input puts the query in a\nsubquery. It adds the cursor condition, the `ORDER BY` and the page\nlimit outside the subquery. Thus, the query needs no placeholder and no\n`ORDER BY`.\n\nWrite one statement with no semicolon at the end. You can use all SQL\nthat the server accepts in a subquery, for example `WITH`.",
"type": [
"string",
"null"
]
},
"table": {
"description": "The table or view to read, as `name` or `schema.name`. Give exactly one\nof `table` and `query`.",
"type": [
"string",
"null"
]
}
},
"required": [
"connection",
"interval_secs",
"mode"
],
"title": "postgres",
"type": "object"
},
"PostgresOutputConfig": {
"description": "Inserts each message in the batch into a postgres table, one row for each\nmessage.\n\nWith `columns`, each entry names a column, its type and the field to read,\nfor example `{\"name\": \"temperature\", \"type\": \"float\", \"field\": \"reading.temp_c\"}`.\nThe default `field` is the name of the column. Without `columns`, the table\nhas an `id`, a `received_at` timestamp and a `payload` column of type\n`jsonb` that contains the full message.\n\nThe output creates the table if it does not exist. Set `create_table` to\nfalse for a table that another system owns. The output does not change an\nexisting table. If the table does not agree with the columns, the insert\nfails with the error from postgres.",
"properties": {
"columns": {
"description": "The column for each message field. If you do not set it, the output\nkeeps each full message as JSON in a `payload` column.",
"items": {
"$ref": "#/$defs/ColumnMapping"
},
"type": "array"
},
"connection": {
"description": "The name of the postgres connection to insert through. Declare the\nconnection in the connections file. The connection gives the host, the\ndatabase and the role.",
"type": "string",
"x-connection": "postgres"
},
"create_table": {
"description": "Create the table on connect if it does not exist. The default is true.",
"type": [
"boolean",
"null"
]
},
"indexes": {
"description": "The indexes to create with the table. Each index names mapped columns,\nin sequence.",
"items": {
"$ref": "#/$defs/TableIndex"
},
"type": "array"
},
"on_extra_fields": {
"$ref": "#/$defs/ExtraFieldPolicy",
"description": "What to do with a message that has fields that no column reads."
},
"primary_key": {
"description": "The columns of the primary key of the created table. If you do not set\nit, the table gets an `id` and a `received_at` timestamp. If you set it,\nthe table does not get these two columns.",
"items": {
"type": "string"
},
"type": "array"
},
"table": {
"description": "The table to insert into. The output creates it if it does not exist.\nYou can add a schema (`analytics.readings`). Use only letters, digits\nand underscores.",
"type": "string"
}
},
"required": [
"connection",
"table"
],
"title": "postgres",
"type": "object"
},
"RecallMissingPolicy": {
"description": "What `recall` does when the bucket has no value for the key of a message.\n\nThe default is `skip`, because the bucket is empty when the pipeline\nstarts. With `error`, the batches fail until the bucket has values.",
"oneOf": [
{
"const": "skip",
"description": "Drop the message. The default. Without the recalled values, a reducer\nafter this transform puts all these messages into one wrong group.",
"type": "string"
},
{
"const": "null",
"description": "Send the message on with the missing names as `null`.",
"type": "string"
},
{
"const": "error",
"description": "Fail the batch. Use it only when another component always fills the\nbucket first.",
"type": "string"
}
]
},
"RecallTransformConfig": {
"description": "Writes values from the state bucket of the pipeline onto each message,\nwith the names from `remember`.\n\nUse it to add a slow fact to a fast stream, for example the current recipe\nof a machine. The values become top-level fields, so a `reducer` after it\ncan group by them.\n\nThe pipeline must have a `state`, or the transform fails to build.",
"properties": {
"on_missing": {
"$ref": "#/$defs/RecallMissingPolicy",
"description": "What to do with a message when the bucket has no value for its key.\nThe default is `skip`."
},
"recall": {
"description": "The names to read from the bucket, as `remember` wrote them. The\ntransform writes each value onto the message with the same name.",
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"recall"
],
"title": "recall",
"type": "object"
},
"RedisConfig": {
"description": "Subscribes to a redis channel and parses each message as JSON.\n\nThe input skips a payload that is not JSON and writes a warning to the log.\nThe input opens the connection on the first read.\n\nThe input uses `SUBSCRIBE`, so the channel name must be exact. Patterns are\nnot supported. Redis pub/sub does not send a message again. When the input\nis not connected, it does not receive the messages that are published.",
"properties": {
"channel": {
"description": "The channel to subscribe to.",
"type": "string"
},
"connection": {
"description": "The name of the redis connection to subscribe on. Declare the\nconnection in the connections file.",
"type": "string",
"x-connection": "redis"
},
"max_batch": {
"description": "The maximum number of messages in one batch. The default is 1.\n\nThe input puts only messages that are already received into a batch.\nIt does not wait for more messages, so a high value does not add\nlatency on a quiet channel.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
}
},
"required": [
"connection",
"channel"
],
"title": "redis",
"type": "object"
},
"RedisOutputConfig": {
"description": "Publishes each message in the batch to a redis channel, one message for\neach publish.",
"properties": {
"channel": {
"description": "The channel to publish to.",
"type": "string"
},
"connection": {
"description": "The name of the redis connection to publish on. Declare the connection\nin the connections file.",
"type": "string",
"x-connection": "redis"
}
},
"required": [
"connection",
"channel"
],
"title": "redis",
"type": "object"
},
"ReduceFnKind": {
"description": "How a reducer combines the values of one field into one result.",
"oneOf": [
{
"const": "sum",
"description": "The total. Numbers only.",
"type": "string"
},
{
"const": "avg",
"description": "The arithmetic mean. Numbers only.",
"type": "string"
},
{
"const": "min",
"description": "The smallest value. Numbers compare as numbers. Strings compare in\nalphabetical sequence, so `min` of ISO timestamps is the earliest time.",
"type": "string"
},
{
"const": "max",
"description": "The largest value. Values compare as for `min`.",
"type": "string"
},
{
"const": "count",
"description": "The number of messages. This function does not need a `field`. With a\n`field`, it counts the messages that have the field.",
"type": "string"
},
{
"const": "count_distinct",
"description": "The number of different values. The function compares the values as\nJSON.",
"type": "string"
},
{
"const": "first",
"description": "The value from the first message of the group, of any type.",
"type": "string"
},
{
"const": "last",
"description": "The value from the last message of the group.",
"type": "string"
},
{
"const": "collect",
"description": "All values as an array, in the sequence of arrival.",
"type": "string"
},
{
"const": "median",
"description": "The middle value, or the mean of the two middle values. Numbers only.",
"type": "string"
},
{
"const": "stddev",
"description": "The population standard deviation. Numbers only.",
"type": "string"
},
{
"const": "slope",
"description": "The rate of change of the field per second, from a least-squares line\nagainst the time of each message. Numbers only. The reducer must have a\n`time` setting, or it fails to build.",
"type": "string"
}
]
},
"ReduceTransformConfig": {
"description": "Reduces a batch to one message for each group. Put a buffer before it, or\nit gets only one message at a time.\n\nWithout `group_by`, the full batch is one group and the reducer sends one\nmessage. With `group_by`, it sends one message for each different\ncombination of those fields. The messages are in the sequence in which the\ngroups first occur. Each message contains the `group_by` fields and the\nresults of the aggregations.\n\nEach aggregation has a `function`, the `field` to use and the name `as` for\nthe result, for example\n`{\"function\": \"avg\", \"field\": \"value\", \"as\": \"mean\"}`.",
"properties": {
"aggregations": {
"description": "The values to calculate. At least one aggregation is required. Each one\nmust have a different `as`.",
"items": {
"$ref": "#/$defs/Aggregation"
},
"type": "array"
},
"group_by": {
"description": "The fields whose combination defines a group. If you do not set it, the\nfull batch is one group.",
"items": {
"type": "string"
},
"type": "array"
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have one of the fields. The\ndefault is `error`."
},
"time": {
"description": "The field that contains the time of each message, as an RFC 3339 string\nor as ms since the epoch. `slope` requires it. A message without this\nfield fails the batch. If you do not set it, the time of each message is\nits arrival time.",
"type": [
"string",
"null"
],
"x-message-field": true
}
},
"required": [
"aggregations"
],
"title": "reducer",
"type": "object"
},
"RememberTransformConfig": {
"description": "Writes values from the messages that match into the state bucket of the\npipeline. The key is the field that `state.key` of the pipeline names.\n\nThe transform sends each message on without changes. It does not filter.\n\nThe pipeline must have a `state`, or the transform fails to build.",
"properties": {
"remember": {
"description": "The values to take from a message that matches. At least one entry is\nrequired. Each entry must have a different `as`.",
"items": {
"$ref": "#/$defs/Remembered"
},
"type": "array"
},
"when": {
"description": "The conditions that a message must match to be remembered. All of them\nmust match. If you do not set it, the transform remembers values from\neach message. Set it when the stream contains more than one type of\nmessage.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"remember"
],
"title": "remember",
"type": "object"
},
"Remembered": {
"description": "One value to write into the state bucket of the pipeline, and its name\nthere.",
"properties": {
"as": {
"description": "The name for the value in the bucket. `recall` reads the value by this\nname. Each entry must have a different name.",
"type": "string"
},
"field": {
"description": "The field to take the value from.",
"type": "string"
}
},
"required": [
"field",
"as"
],
"type": "object"
},
"ResampleMethod": {
"description": "How the transform calculates the value at a grid point from the readings in\nits interval, and what it does with an empty interval.",
"oneOf": [
{
"const": "last",
"description": "The last reading in the interval. An empty interval sends nothing.",
"type": "string"
},
{
"const": "mean",
"description": "The mean of the readings in the interval. An empty interval sends\nnothing.",
"type": "string"
},
{
"const": "linear",
"description": "The value on a straight line between the last reading before the grid\npoint and the first reading after it. The transform sends the grid\npoint when the reading after it arrives. The same line fills the empty\nintervals between the two readings.",
"type": "string"
},
{
"const": "forward_fill",
"description": "The last reading, carried forward. An empty interval repeats the last\nvalue for a maximum of `max_gap_seconds`, and then stops. This is the\nonly method that sends values from a quiet series. Use it to change a\nsignal that reports on change into a regular signal.",
"type": "string"
}
]
},
"ResampleTransformConfig": {
"description": "Puts a series onto a regular grid. The transform sends one message for each\nkey in each `interval_seconds`, at times that are multiples of the interval.\nThe rate of the readings has no effect on the grid.\n\nThe message that the transform sends contains:\n\n- the group fields, under their leaf names,\n- the grid time as an RFC 3339 string, under the name of the `time` field\n (or `time` when the transform uses the arrival time),\n- the value, under `as` (the leaf of `field` when you leave it out).\n\nThe transform sends a grid point when a later reading arrives. With\n`forward_fill` and the arrival time, it also sends a grid point when the\nclock passes it.",
"properties": {
"as": {
"description": "The field to write the value to. The default is the leaf of `field`.",
"type": [
"string",
"null"
]
},
"field": {
"description": "The numeric field to resample.",
"type": "string",
"x-message-field": true
},
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out for\none series.",
"items": {
"type": "string"
},
"type": "array"
},
"interval_seconds": {
"description": "The distance between two grid points, in seconds.",
"format": "double",
"type": "number"
},
"max_gap_seconds": {
"description": "For `forward_fill`: the maximum time to repeat a value into empty\nintervals, in seconds. Leave it out to repeat the value with no limit.",
"format": "double",
"type": [
"number",
"null"
]
},
"method": {
"$ref": "#/$defs/ResampleMethod",
"description": "How the transform calculates the value of an interval."
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have the field or a group\nfield. The default is `error`."
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"time": {
"description": "The field that holds the time of each message, as an RFC 3339 string\nor as milliseconds since the epoch. Leave it out to use the arrival\ntime. With the arrival time, the clock finds the empty intervals.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"field",
"interval_seconds",
"method"
],
"title": "resample",
"type": "object"
},
"RollingTransformConfig": {
"description": "Writes aggregations over the last messages of the series onto each\nmessage. The aggregations use the same `{function, field, as}` list and\nfunctions as `reduce`. One message goes in and one message comes out.\n\nThe window holds a maximum of `size` messages. `size` is always required.\nWith `seconds`, the window also drops messages that are older than that\ntime, from the `time` field.\n\nHere, `count` needs a `field`. It counts the messages in the window that\nhave the field. When the window is full, the count is `size`.",
"properties": {
"aggregations": {
"description": "The aggregations to calculate over the window. Give one or more, each\nwith a different `as`.",
"items": {
"$ref": "#/$defs/Aggregation"
},
"type": "array"
},
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out for\none series.",
"items": {
"type": "string"
},
"type": "array"
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have an aggregated field or a\ngroup field. The default is `error`."
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"seconds": {
"description": "Also drop the messages that are older than this number of seconds,\nfrom the `time` field.",
"format": "double",
"type": [
"number",
"null"
]
},
"size": {
"description": "The maximum number of messages in the window.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"time": {
"description": "The field that holds the time of each message, as an RFC 3339 string\nor as milliseconds since the epoch. Leave it out to use the arrival\ntime.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"aggregations",
"size"
],
"title": "rolling",
"type": "object"
},
"RotationConfig": {
"description": "When the output closes a file and starts the next file.\n\nThe two triggers are optional. The first trigger that is reached rotates\nthe file. With no trigger, the output writes one file while the pipeline\nruns. The `file` and `s3` outputs use the same rotation settings.",
"properties": {
"interval_secs": {
"description": "Close the file this number of seconds after the output opened it. The\ntime starts when the file opens, not at the last write.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"max_rows": {
"description": "Close the file when it contains this number of messages. The output\ndoes not divide a batch, so a file can contain more messages.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
}
},
"type": "object"
},
"RunStatus": {
"description": "The status of the run loop of a pipeline.",
"oneOf": [
{
"const": "starting",
"description": "The run loop started, and an output is not initialized yet. For\nexample, the database of an output is not available. The pipeline tries\nagain with a backoff. When the output initializes, the status changes\nto `running`.",
"type": "string"
},
{
"const": "running",
"description": "All outputs are initialized, and the loop runs. Messages move only in\nthis status.",
"type": "string"
},
{
"const": "stopped",
"description": "The loop stopped because it was cancelled, by a delete, a revert or a\nshutdown. This status is rare in a response, because kayak usually\nremoves the pipeline at the same time.",
"type": "string"
},
{
"const": "failed",
"description": "The loop stopped because the last input failed. The pipeline sends\nnothing more until kayak builds it again.",
"type": "string"
}
]
},
"S3OutputConfig": {
"description": "Writes each batch to objects under a prefix in an S3-compatible bucket.\n\nThe `s3` output uses the same file names, formats and rotation as the\n`file` output. An object store cannot append to an object. Thus the output\nkeeps the current object in memory and uploads it when it rotates.\n`rotate` is required. It sets how frequently objects appear and how much\nmemory the pipeline uses.",
"properties": {
"connection": {
"description": "The name of the s3 connection to write through. Declare the connection\nin the connections file. The connection gives the bucket and the\ncredentials.",
"type": "string",
"x-connection": "s3"
},
"format": {
"anyOf": [
{
"$ref": "#/$defs/FileFormat"
},
{
"type": "null"
}
],
"description": "The layout of the messages. The default is `ndjson`."
},
"prefix": {
"description": "The key prefix to write under, for example `orders`. The output writes\neach object to `<prefix>/<part name>`. Set an empty prefix to write at\nthe root of the bucket.",
"type": "string"
},
"rotate": {
"$ref": "#/$defs/RotationConfig",
"description": "When to close an object and start the next object. Required. A\n`rotate` with no trigger fails to build."
}
},
"required": [
"connection",
"prefix",
"rotate"
],
"title": "s3",
"type": "object"
},
"ScriptScope": {
"description": "Whether the script gets one message or the full batch.\n\n`message` is the default. The operation limit then applies to each message,\nand the batch keeps its structure.\n\nUse `batch` for work on the full batch. For example, remove duplicates in\nthe batch, or calculate a value that `reduce` has no function for.",
"oneOf": [
{
"const": "message",
"description": "The script runs one time for each message, with the message in `msg`.",
"type": "string"
},
{
"const": "batch",
"description": "The script runs one time for each batch, with the messages in `batch`\nas an array. When you emit an array, the transform sends a batch of\nthose messages.",
"type": "string"
}
]
},
"ScriptSource": {
"description": "The source of the script text: `inline` or `file`.\n\nUse `inline` to send a script through the HTTP API. In a YAML config, an\ninline script is a literal block. Use `file` to keep the script in its own\nfile, where an editor, a formatter and a test can use it. A save keeps the\n`file` reference in the config. It does not copy the script into the\nconfig.",
"oneOf": [
{
"description": "The script text, in the config. Use a YAML config for an inline script.\nYAML keeps the script as a literal block. JSON must escape each newline.",
"properties": {
"code": {
"description": "The rhai source.",
"type": "string",
"x-script": "rhai"
},
"type": {
"const": "inline",
"type": "string"
}
},
"required": [
"type",
"code"
],
"type": "object"
},
{
"description": "A path to a `.rhai` file, relative to the directory of the config file.\n\nkayak reads the file and its imports when it builds the pipeline. After\nyou change the file, do a revert to use the change. A server with no\nconfig file does not accept a `file` source. Inline scripts work on that\nserver, but their imports do not.",
"properties": {
"path": {
"description": "The path, relative to the directory of the config file. The path\nmust stay in that directory.",
"type": "string"
},
"type": {
"const": "file",
"type": "string"
}
},
"required": [
"type",
"path"
],
"type": "object"
}
]
},
"ScriptTransformConfig": {
"description": "Runs a [rhai](https://rhai.rs) script over each message, or over the full\nbatch, and sends the values that the script emits.\n\nThe script gets the message as `msg`, and sends values with `emit(value)`.\nCall `emit` zero times to drop the message, one time to replace it, or more\ntimes to split it. Use a script when `filter`, `map` and `splitter` are not\nsufficient.\n\nkayak **compiles the script when it builds the pipeline**. Thus, a syntax\nerror stops the pipeline from starting. An error that occurs only with a\nmessage fails that batch. For example, a missing field or a value that does\nnot convert fails the batch.\n\nEach run of the script has an **operation limit** (`max_operations`). When\nthe script reaches the limit, kayak stops it and fails the batch. Thus, a\nscript with an endless loop cannot block the pipeline.\n\nA script can **`import`** other rhai files. Give a literal path relative to\nthe directory of the config file. The path must stay in that directory. You\ncan leave out the `.rhai` extension. kayak reads the imports when it builds\nthe pipeline. Thus, a bad import stops the pipeline from starting, and a\nrunning script does not read the filesystem.",
"properties": {
"max_operations": {
"description": "The maximum number of rhai operations in one run of the script. At the\nlimit, kayak stops the script and fails the batch. The default is\n100000. Increase it for a script that walks a large array.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"scope": {
"$ref": "#/$defs/ScriptScope",
"description": "Whether the script gets one message at a time or the full batch. The\ndefault is `message`."
},
"source": {
"$ref": "#/$defs/ScriptSource",
"description": "The script: inline in the config, or in a file beside the config."
}
},
"required": [
"source"
],
"title": "script",
"type": "object"
},
"Secret": {
"description": "A text value that can refer to secrets with `${NAME}` references.\n\nThe value is an ordinary JSON string. When kayak builds the pipeline, it\nreplaces each `${NAME}` reference with the value from the secret store of\nthe server:\n\n```json\n{ \"type\": \"nats\", \"urls\": \"nats://app:${NATS_PASSWORD}@broker:4222\" }\n```\n\nThe config keeps only the reference, never the value. Thus you can commit\nthe file, and `GET /api/pipelines` does not show the secret.\n\nkayak uses a value with no `${...}` reference as it is.",
"type": "string"
},
"SmoothMethod": {
"description": "How the transform smooths a value against the values before it.",
"oneOf": [
{
"description": "An exponentially weighted moving average. It needs no window. Give\nexactly one of `alpha`, `half_life` or `tau_seconds`.\n\n`alpha` and `half_life` count messages. Use them when messages arrive\nat a steady rate. `tau_seconds` counts time. The weight of a value is\n`1 − e^(−Δt/τ)`, where Δt is the time since the previous value. Thus, a\nreading after a long gap has more weight. Use `tau_seconds` for a sensor\nthat reports on change, or for a stream that stops for periods.",
"properties": {
"alpha": {
"description": "The weight of the newest value, from 0 to 1.",
"format": "double",
"type": [
"number",
"null"
]
},
"half_life": {
"description": "The number of messages after which the weight of a value is half.",
"format": "double",
"type": [
"number",
"null"
]
},
"tau_seconds": {
"description": "The time constant in seconds. After this time, the weight of an old\nvalue is about 37%. It reads the `time` field of the transform.",
"format": "double",
"type": [
"number",
"null"
]
},
"type": {
"const": "ewma",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"description": "The median of the last `size` values, with this value included. It\nremoves a spike of one sample.",
"properties": {
"size": {
"description": "The number of values in the window.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"type": {
"const": "median",
"type": "string"
}
},
"required": [
"type",
"size"
],
"type": "object"
},
{
"description": "A Hampel filter. When the value is more than `threshold` robust\nstandard deviations from the median of the window, the median replaces\nit. Otherwise the value stays. Use it before `detect` to remove\noutliers.",
"properties": {
"size": {
"description": "The number of values in the window, with this value included.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"threshold": {
"description": "The number of scaled MADs from the median that makes an outlier.\nThe default is `3`.",
"format": "double",
"type": [
"number",
"null"
]
},
"type": {
"const": "hampel",
"type": "string"
}
},
"required": [
"type",
"size"
],
"type": "object"
},
{
"description": "A Savitzky–Golay filter. It fits a polynomial of degree `order` to the\nlast `size` values by least squares, and gives its value at the newest\npoint. It keeps the shape of peaks. The window uses only earlier\nvalues. Until the window holds more than `order` values, the value\npasses unchanged.",
"properties": {
"order": {
"description": "The degree of the polynomial. It must be less than `size`. The\ndefault is `2`.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"size": {
"description": "The number of values in the window, with this value included.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"type": {
"const": "savitzky_golay",
"type": "string"
}
},
"required": [
"type",
"size"
],
"type": "object"
}
]
},
"SmoothTransformConfig": {
"description": "Smooths a numeric field against the earlier values of its series. The\ntransform writes the result into the field, or under `as`.\n\nAll methods except `ewma` with `tau_seconds` use the order of the values,\nand not their time. Thus, `time` is permitted only with `ewma` and\n`tau_seconds`. With other methods, `time` is an error when the pipeline\nbuilds.",
"properties": {
"as": {
"description": "The field to write the smoothed value to. Leave it out to replace the\nvalue of `field`.",
"type": [
"string",
"null"
]
},
"field": {
"description": "The numeric field to smooth.",
"type": "string",
"x-message-field": true
},
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out for\none series.",
"items": {
"type": "string"
},
"type": "array"
},
"method": {
"$ref": "#/$defs/SmoothMethod",
"description": "The method to smooth the value with."
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have the field or a group\nfield. The default is `error`."
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"time": {
"description": "For `ewma` with `tau_seconds` only: the field that holds the time of\neach message, as an RFC 3339 string or as milliseconds since the epoch.\nLeave it out to use the arrival time.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"field",
"method"
],
"title": "smooth",
"type": "object"
},
"SplitterTransformConfig": {
"description": "Divides one batch into smaller batches.\n\nThe last batch contains the messages that remain and can be smaller. For\nexample, 4 messages with `out_size: 3` give a batch of 3 and a batch of 1.",
"properties": {
"out_size": {
"description": "The number of messages in each sent batch.",
"format": "uint",
"minimum": 0,
"type": "integer"
}
},
"required": [
"out_size"
],
"title": "splitter",
"type": "object"
},
"StartFrom": {
"description": "Where the first read of an incremental input starts.",
"oneOf": [
{
"const": "oldest",
"description": "From the start. The first read returns all rows, one page at a time.",
"type": "string"
},
{
"const": "newest",
"description": "From now. The first read finds the highest value of the field and\nreturns only the rows above it. This is the default.",
"type": "string"
}
]
},
"StdoutOutputConfig": {
"description": "Prints each batch to the standard output of the server. Use it to test a\npipeline. It has no settings.",
"title": "stdout",
"type": "object"
},
"StringFilterOperatorKind": {
"description": "How a `string` condition compares a string to the value in the config.",
"enum": [
"equal_to",
"not_equal_to",
"contains"
],
"type": "string"
},
"TableIndex": {
"description": "An index to make with the table. The output makes the index only when it\nmakes the table, with `IF NOT EXISTS`.",
"properties": {
"columns": {
"description": "The columns to index, in order. Each column must be a mapped column.",
"items": {
"type": "string"
},
"type": "array"
},
"unique": {
"description": "Whether the index is unique. The default is false.",
"type": [
"boolean",
"null"
]
}
},
"required": [
"columns"
],
"title": "index",
"type": "object"
},
"ThrottleTransformConfig": {
"description": "Passes a maximum of one message per key in each period of `seconds`, and\ndrops the other messages.\n\nThe first message for each key passes. The next message that passes is the\nfirst one that arrives `seconds` or more after it. The transform drops all\nmessages between them. The period starts at the message that passed. It is\nnot aligned to the clock.\n\nThe transform does not keep messages to send later. A key that becomes\nquiet sends nothing until its next message. Use `resample` when you need the\nlast value of each period, or a value from a quiet key.\n\n`throttle` does not read a value. It passes complete messages with all\ntheir fields.",
"properties": {
"group_by": {
"description": "The fields that identify a series, as in `reduce`. Leave it out for\none series.",
"items": {
"type": "string"
},
"type": "array"
},
"on_missing": {
"$ref": "#/$defs/MissingFieldPolicy",
"description": "What to do with a message that does not have a group field. The\ndefault is `error`."
},
"reset_when": {
"description": "The conditions that clear the state of the message's key. All of them\nmust match. The series then starts again. The transform checks these\nbefore `when`.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
},
"seconds": {
"description": "The minimum time between two messages that pass for one key, in\nseconds.",
"format": "double",
"type": "number"
},
"time": {
"description": "The field that holds the time of each message, as an RFC 3339 string\nor as milliseconds since the epoch. Leave it out to use the arrival\ntime.",
"type": [
"string",
"null"
],
"x-message-field": true
},
"when": {
"description": "The conditions that a message must match to be applied. All of them\nmust match. Other messages pass through unchanged. Leave it out to\napply every message.",
"items": {
"$ref": "#/$defs/Condition"
},
"type": "array"
}
},
"required": [
"seconds"
],
"title": "throttle",
"type": "object"
},
"TidepoolOutputConfig": {
"description": "Writes each batch into a Tidepool table, one request for each batch.\n\nThe table must exist in the Tidepool project. On start, the output checks\nthe columns against the table. Each mapped column must be in the table and\nhave a type that the mapping can write. Each required column of the table\nmust be written. If the check fails, the start fails.\n\n`columns` has the same format as on the database outputs. If you do not set\nit, the output sends each message as a row without changes. Tidepool checks\neach value and refuses a batch that has a problem.\n\nA refused batch fails. The error gives the problems by row and column. The\noutput tries again when the server is busy (`503`), when the server cannot\nbe reached and on other `5xx` errors. It tries again for up to\n`retry_seconds`. Each try uses the same idempotency key, so Tidepool does\nnot write a batch two times.",
"properties": {
"columns": {
"description": "The column for each message field. If you do not set it, the output\nsends each message as a row without changes.",
"items": {
"$ref": "#/$defs/ColumnMapping"
},
"type": "array"
},
"connection": {
"description": "The name of the tidepool connection to write through. Declare the\nconnection in the connections file.",
"type": "string",
"x-connection": "tidepool"
},
"on_extra_fields": {
"$ref": "#/$defs/ExtraFieldPolicy",
"description": "What to do with a message that has fields that no column reads."
},
"retry_seconds": {
"description": "The maximum time to try one batch again while the server is busy or\ncannot be reached, in s. The default is 30.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"table": {
"description": "The table to write into, with the name from the Tidepool project.",
"type": "string"
},
"timeout_seconds": {
"description": "The maximum time for one request, in s. The default is 30.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
}
},
"required": [
"connection",
"table"
],
"title": "tidepool",
"type": "object"
},
"TimeFormat": {
"description": "How a [`Mapping::TimeBucket`] writes a time.",
"oneOf": [
{
"const": "rfc3339",
"description": "An RFC 3339 string in UTC, with milliseconds. kayak writes all times in\nthis format.",
"type": "string"
},
{
"const": "millis",
"description": "Milliseconds since the epoch, as a number.",
"type": "string"
}
]
},
"TransformConfig": {
"oneOf": [
{
"$ref": "#/$defs/BufferTransformConfig",
"properties": {
"type": {
"const": "buffer",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/HttpTransformConfig",
"properties": {
"type": {
"const": "http",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/SplitterTransformConfig",
"properties": {
"type": {
"const": "splitter",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/ReduceTransformConfig",
"properties": {
"type": {
"const": "reducer",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/FilterTransformConfig",
"properties": {
"type": {
"const": "filter",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/RememberTransformConfig",
"properties": {
"type": {
"const": "remember",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/RecallTransformConfig",
"properties": {
"type": {
"const": "recall",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/MapTransformConfig",
"properties": {
"type": {
"const": "map",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/ScriptTransformConfig",
"properties": {
"type": {
"const": "script",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/DeadbandTransformConfig",
"properties": {
"type": {
"const": "deadband",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/ThrottleTransformConfig",
"properties": {
"type": {
"const": "throttle",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/PivotTransformConfig",
"properties": {
"type": {
"const": "pivot",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/DeriveTransformConfig",
"properties": {
"type": {
"const": "derive",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/RollingTransformConfig",
"properties": {
"type": {
"const": "rolling",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/SmoothTransformConfig",
"properties": {
"type": {
"const": "smooth",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/DetectTransformConfig",
"properties": {
"type": {
"const": "detect",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/ResampleTransformConfig",
"properties": {
"type": {
"const": "resample",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/FeaturesTransformConfig",
"properties": {
"type": {
"const": "features",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
}
],
"type": "object"
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "One pipeline as the API reports it: the id that it runs under, the config\nthat kayak built it from, and the status of its run loop.",
"properties": {
"config": {
"$ref": "#/$defs/Config"
},
"id": {
"type": "string"
},
"status": {
"$ref": "#/$defs/RunStatus",
"default": "running",
"description": "The status of the run loop. When it is absent, read it as `running`."
}
},
"required": [
"id",
"config"
],
"title": "PipelineDto",
"type": "object"
}PipelineHistory
What GET /api/pipelines/{id}/history answers with.
schema
json
{
"$defs": {
"ErrorSignature": {
"description": "One distinct failure, and how it has behaved over time.\n\n**Aggregated rather than logged**, which is the difference between a useful\nmorning readout and two million rows to scroll. A pipeline whose broker went\ndown at 02:14 and stayed down is one of these saying so, with a count — and\nthat is both cheaper to keep and easier to read than the log it replaces.\n\nIdentity is (`stage`, `component`, `message`): the same text from the second\nof two outputs is a different fact from the first one's, which is the same\nrule the run loop's failure budget already uses.",
"properties": {
"component": {
"description": "Which component of that stage, indexed into its array in the config.\n`None` where the run loop doesn't know — an input failure, since inputs\nare merged before the loop sees them.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"count": {
"description": "How many times it has happened, including the repeats the log\nsuppressed. See [`HistoryBucket::errors`] — same accounting.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"first_seen": {
"description": "When it was first seen, in milliseconds since the epoch. This is the\nnumber the morning question is actually about.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"last_seen": {
"description": "When it was last seen. Equal to `first_seen` for a one-off; far from it\nfor something still broken, which is how the two are told apart.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"message": {
"description": "The failure's text, as the log line would have shown it, cut to\n[`crate::MAX_MESSAGE_BYTES`]. Cut rather than kept whole for the reason\nthe feed's messages are: an error with a payload embedded in it can be\narbitrarily long, and this is a store that promises to be bounded.",
"type": "string"
},
"stage": {
"$ref": "#/$defs/Stage",
"description": "Where in the pass it failed."
}
},
"required": [
"stage",
"message",
"first_seen",
"last_seen",
"count"
],
"type": "object"
},
"HistoryBucket": {
"description": "One time unit's worth of counting, as the store keeps it and the API serves\nit.\n\nDeliberately the same three questions the card chart asks\n(`frontend::stats::Bucket`) plus the one it can't answer from a sampled\nfeed: how many failures there were. Counts, not messages — which is what\nmakes a bucket 32 bytes whatever the pipeline is carrying, and what makes\nkeeping a day of them cost less than one message of most real payloads.",
"properties": {
"errors": {
"description": "Failures at any stage during the bucket. The *true* count, not the\nthrottled one: this is a counter, so suppressing a repeat in the log\ndoesn't hide it here.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"inbound": {
"description": "Messages that arrived at the pipeline's inputs during the bucket.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"outbound": {
"description": "Messages that came out of the transform chain and were handed to the\noutputs during the bucket.\n\nCounted once per batch, not once per output — it is what the pipeline\n*produced*, which is the same thing the card chart's outbound bar counts\noff the `Stage::Output` events, so the two agree. A transform that\nchanges cardinality is what makes this differ from `inbound`; a failing\noutput is not, and shows up in `errors` instead.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"start": {
"description": "Where the bucket starts, in seconds since the epoch. Always a multiple\nof the resolution's width.",
"format": "uint64",
"minimum": 0,
"type": "integer"
}
},
"required": [
"start",
"inbound",
"outbound",
"errors"
],
"type": "object"
},
"Resolution": {
"description": "Which ring a query is asking for. See the module docs for why there are two.",
"oneOf": [
{
"const": "fine",
"description": "[`FINE_BUCKET_SECS`] a bucket, over [`FINE_WINDOW_SECS`]. What a card\nbackfills its live chart from.",
"type": "string"
},
{
"const": "coarse",
"description": "[`COARSE_BUCKET_SECS`] a bucket, over the configured retention. The\novernight record, and the default because that is what someone asking\nfor history at all is usually asking for.",
"type": "string"
}
]
},
"Stage": {
"description": "The stage of the run loop that an event comes from.",
"enum": [
"input",
"transform",
"output"
],
"type": "string"
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "What `GET /api/pipelines/{id}/history` answers with.",
"properties": {
"bucket_secs": {
"description": "How wide one bucket is, in seconds. Derivable from `resolution`, sent\nanyway so a chart can scale its axis without a table of constants.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"buckets": {
"description": "Oldest first, contiguous — gaps are filled with empty buckets rather\nthan omitted, so \"the pipeline stopped\" and \"the server wasn't asked\"\ndon't look alike. Empty when the pipeline has produced nothing yet.",
"items": {
"$ref": "#/$defs/HistoryBucket"
},
"type": "array"
},
"dropped_signatures": {
"description": "Distinct failures dropped to stay under [`MAX_ERROR_SIGNATURES`], since\nthis pipeline started. Non-zero means the errors above are a selection,\nand is itself a diagnosis: a pipeline producing dozens of distinct\nfailure texts is usually one embedding a message id in each.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"errors": {
"description": "Distinct failures, most recently seen first, at most\n[`MAX_ERROR_SIGNATURES`] of them. Not scoped to the buckets' window:\na failure that started before the window is exactly the one worth\nshowing, and `first_seen` says so.",
"items": {
"$ref": "#/$defs/ErrorSignature"
},
"type": "array"
},
"resolution": {
"$ref": "#/$defs/Resolution",
"description": "Which ring this came from, echoed so a client that took the default\nknows what it got."
}
},
"required": [
"resolution",
"bucket_secs",
"buckets",
"errors",
"dropped_signatures"
],
"title": "PipelineHistory",
"type": "object"
}PipelineSource
One running pipeline's config, written out the way a config file would write it — what a card's source view shows and copies.
schema
json
{
"$defs": {
"ConfigFormat": {
"description": "The two ways a config file can be written.\n\nJSON is the default because it is what every existing file and every example\nin the repository uses; a file only gets read as YAML if it says so.",
"enum": [
"json",
"yaml"
],
"type": "string"
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "One running pipeline's config, written out the way a config file would\nwrite it — what a card's source view shows and copies.",
"properties": {
"format": {
"$ref": "#/$defs/ConfigFormat",
"description": "The format `text` is in: the one asked for, or else the config file's."
},
"text": {
"description": "The pipeline as one entry of a config file would hold it, `id` filled\nin.",
"type": "string"
}
},
"required": [
"format",
"text"
],
"title": "PipelineSource",
"type": "object"
}SampleRequest
Take a few messages from an input, without creating a pipeline.
schema
json
{
"$defs": {
"AckMode": {
"description": "When an input acknowledges a message to its broker.",
"oneOf": [
{
"const": "on_receipt",
"description": "Acknowledge the message when it arrives, before the transforms and the\noutputs. The default. A crash before the output writes the message can\nlose it.",
"type": "string"
},
{
"const": "on_delivery",
"description": "Acknowledge the message when it leaves this pipeline. Each output of\nthis pipeline must return, with or without success. Each downstream\npipeline must accept the message into its queue. A failed output does\nnot stop the acknowledgement. kayak does not wait for the outputs of the\ndownstream pipelines.",
"type": "string"
}
]
},
"BufferConfig": {
"description": "How an input collects its messages into batches before the transforms.\n\nThe three types use two limits: a count, a time, or both. With both, the\nfirst limit that is reached closes the batch. A buffer never sends an empty\nbatch. The time starts when the first message of a batch arrives, so a\nquiet input sends nothing.\n\n`size` is a minimum, not a maximum. The buffer does not divide a batch\nthat arrives. Thus an input with `max_batch` can give a larger batch.",
"oneOf": [
{
"description": "Wait for a number of messages. There is no time limit.",
"properties": {
"size": {
"description": "The number of messages in a batch.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"type": {
"const": "static",
"type": "string"
}
},
"required": [
"type",
"size"
],
"type": "object"
},
{
"description": "Wait for a time. The batch contains at least one message. The time\nstarts when the first message arrives.",
"properties": {
"type": {
"const": "tumbling",
"type": "string"
},
"window_seconds": {
"description": "The time to collect messages, in s, from the first message.",
"format": "uint",
"minimum": 0,
"type": "integer"
}
},
"required": [
"type",
"window_seconds"
],
"type": "object"
},
{
"description": "Use both limits. The first limit that is reached closes the batch. Use\nthis type when the rate of the input changes. `size` sets the largest\nbatch when the input is busy. `window_seconds` sets the longest wait\nwhen the input is quiet.",
"properties": {
"size": {
"description": "The number of messages that closes the batch immediately.",
"format": "uint",
"minimum": 0,
"type": "integer"
},
"type": {
"const": "batch",
"type": "string"
},
"window_seconds": {
"description": "The maximum time to wait, in s, from the first message in the batch.",
"format": "uint",
"minimum": 0,
"type": "integer"
}
},
"required": [
"type",
"size",
"window_seconds"
],
"type": "object"
}
]
},
"ClickhouseInputConfig": {
"description": "Reads a `ClickHouse` table, view or query at an interval and sends each row\nas a message. It uses the HTTP interface of `ClickHouse`.\n\n`ClickHouse` sends the rows as `JSONEachRow`. A `DateTime` is ISO 8601, an\n`Int64` is a number, and a `Decimal` keeps its digits. The modes, the\nwatermark and the limits of an incremental read are the same as on the\n`postgres` input.",
"properties": {
"columns": {
"description": "The columns to read, in order. Leave it empty to read all columns. For\nan incremental input, the list must include `field`.",
"items": {
"type": "string"
},
"type": "array"
},
"connection": {
"description": "The name of the clickhouse connection to read through. Declare the\nconnection in the connections file.",
"type": "string",
"x-connection": "clickhouse"
},
"interval_secs": {
"description": "The time between two reads, in seconds. The time starts at the end of a\nread. Thus, two reads never overlap. The first read occurs when the\npipeline starts.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"max_batch": {
"description": "The maximum number of rows in one batch. The default is 1. The input\nputs rows that it already read into a batch. It does not wait for more\nrows to fill a batch.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"mode": {
"$ref": "#/$defs/PollMode",
"description": "Whether each read returns all rows (`snapshot`) or only the rows after\nthe last read (`incremental`)."
},
"page_size": {
"description": "The maximum number of rows that one query returns. The default is\n1000. When a page is full, the input reads the next page immediately.\nThe interval starts after a page that is not full. A `snapshot` ignores\nthis field and reads all rows in one query.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"query": {
"description": "A `SELECT` to read in place of a table. The input puts the query in a\nsubquery. It adds the cursor condition, the `ORDER BY` and the page\nlimit outside the subquery. Thus, the query needs no placeholder and no\n`ORDER BY`.\n\nWrite one statement with no semicolon at the end. You can use all SQL\nthat the server accepts in a subquery, for example `WITH`.",
"type": [
"string",
"null"
]
},
"table": {
"description": "The table or view to read, as `name` or `schema.name`. Give exactly one\nof `table` and `query`.",
"type": [
"string",
"null"
]
}
},
"required": [
"connection",
"interval_secs",
"mode"
],
"title": "clickhouse",
"type": "object"
},
"DummyConfig": {
"description": "Sends one generated message at a fixed interval. Use it to test a pipeline\nwithout a real source.\n\nEach message contains a `value` and the `current_time` at which the input\nsent it. The `payload` field sets the type of `value`. It can be a number\nfrom a sine wave or a random sentence.",
"properties": {
"amplitude": {
"description": "The peak of the sine wave. The value goes from `-amplitude` to\n`+amplitude`. Applies only to the `number` payload. The default is 1.",
"format": "double",
"type": [
"number",
"null"
]
},
"duration": {
"description": "The time between two messages, in s.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"payload": {
"anyOf": [
{
"$ref": "#/$defs/DummyPayload"
},
{
"type": "null"
}
],
"description": "The type of `value` in each message. `number` is a number from a sine\nwave. `text` is a random sentence. The default is `number`."
},
"period": {
"description": "The time for one full cycle of the sine wave, in s. Applies only to the\n`number` payload. The default is 60. The input samples the wave by the\nclock, so the period does not change with `duration`.",
"format": "double",
"type": [
"number",
"null"
]
}
},
"required": [
"duration"
],
"title": "dummy",
"type": "object"
},
"DummyPayload": {
"description": "The type of `value` in each message from a `dummy` input.",
"oneOf": [
{
"const": "number",
"description": "A number from a sine wave.",
"type": "string"
},
{
"const": "text",
"description": "A random sentence.",
"type": "string"
}
]
},
"EnvelopeConfig": {
"description": "How an input adds metadata about the source of each message.\n\nThe \"metadata\" section of each input lists its metadata. Examples are the\nsubject of a nats message and the topic, partition and offset of a kafka\nrecord. The metadata also contains the pipeline and the input type.\n\nThe input adds the metadata as ordinary fields on the message. Thus each\ntransform can use it as it uses the fields of the payload, for example\n`\"group_by\": [\"_meta.subject\"]`.\n\nIf you do not set an envelope, the input sends each message without\nchanges. An envelope changes the shape of each message from the input.\nUpdate the field paths downstream when you add one.",
"oneOf": [
{
"description": "Add the metadata as one more field on the message. The fields of the\npayload do not move.\n\nThis type works only on a payload that is a JSON object. The input\nskips a message that is a number or a string and writes a warning to\nthe log. Use `wrap` for these payloads.",
"properties": {
"meta": {
"description": "The field for the metadata object. The default is `_meta`.",
"type": [
"string",
"null"
]
},
"type": {
"const": "merge",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"description": "Put the full payload under a field, next to the metadata:\n`{\"value\": …, \"_meta\": {…}}`.\n\nThis type works with all payloads, for example a `1` or a `\"recipe-a\"`.\nEach field path downstream must then start with the payload field, for\nexample `value.temperature`.",
"properties": {
"meta": {
"description": "The field for the metadata object. The default is `_meta`.",
"type": [
"string",
"null"
]
},
"payload": {
"description": "The field for the original payload. The default is `value`.",
"type": [
"string",
"null"
]
},
"type": {
"const": "wrap",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
}
]
},
"HttpAuthConfig": {
"description": "A credential in a header. The `http` input checks it on each post. The\n`http` output, the `http` transform and the `http_poll` input send it on\neach request.\n\nThis credential is for one pipeline only. It is not related to the user\naccounts of the server.\n\nThe sender sends the same token on each request. Thus the token is only as\nsecure as the connection. kayak serves plain HTTP. Put TLS in front of\nkayak, or other systems on the network path can read the token.",
"oneOf": [
{
"description": "A token in the standard `Authorization` header, as\n`Authorization: Bearer TOKEN`. Use this variant if the other system can\nuse that header.",
"properties": {
"token": {
"$ref": "#/$defs/Secret",
"description": "The token. Use a `${NAME}` reference, so that the config file keeps\nonly the name and the secret store keeps the value."
},
"type": {
"const": "bearer",
"type": "string"
}
},
"required": [
"type",
"token"
],
"type": "object"
},
{
"description": "A fixed value in a header that you name. Use this variant for a system\nthat cannot use the `Authorization` header.",
"properties": {
"name": {
"description": "The name of the header. The `http` input compares the name without\ncase. On an `http` input, the name must not be a header that an\n`envelope` copies into the messages.",
"type": "string"
},
"type": {
"const": "header",
"type": "string"
},
"value": {
"$ref": "#/$defs/Secret",
"description": "The exact value of the header. Use a `${NAME}` reference."
}
},
"required": [
"type",
"name",
"value"
],
"type": "object"
}
]
},
"HttpInputConfig": {
"description": "Accepts messages that are posted to the endpoint of the pipeline,\n`POST /api/pipelines/{id}/messages`.\n\nkayak makes the endpoint from the pipeline id. The endpoint is available\nwhen the pipeline runs. The body is one JSON message or an array of\nmessages. An array becomes one batch. A pipeline can have only one `http`\ninput. A second `http` input fails to build.",
"properties": {
"auth": {
"anyOf": [
{
"$ref": "#/$defs/HttpAuthConfig"
},
{
"type": "null"
}
],
"description": "The credential that a post must have. If you do not set it, the\nendpoint accepts all posts. A post without the correct credential gets\n`401`."
},
"capacity": {
"description": "The maximum number of posted batches in the queue before the pipeline.\nThe default is 1024. When the queue is full, the endpoint refuses a\npost with `503`.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
}
},
"title": "http",
"type": "object"
},
"HttpPollConfig": {
"description": "Gets a url at an interval and sends the full reply each time.\n\nEach read is a `GET`. A reply that is an array gives one message for each\nelement. Any other reply gives one message. Use `items` for a reply that\nhas the records inside it, for example `{\"data\": {\"machines\": [...]}}`.\n\nUse this input for reference data, for example a list of machines or\nrecipes that changes rarely. The input has no watermark and no pages. Each\nread sends all records again. Send them to an output that writes the latest\nvalue for each key.\n\nA read fails when the url cannot be reached, when the status is not 2xx,\nwhen the body is not JSON, or when `items` finds nothing. The input reports\nthe failure one time and tries again with backoff. The interval starts\nagain after the next successful read.",
"properties": {
"auth": {
"anyOf": [
{
"$ref": "#/$defs/HttpAuthConfig"
},
{
"type": "null"
}
],
"description": "The credential that the input sends. If you do not set it, the input\nsends no credential."
},
"interval_secs": {
"description": "The time between two reads, in s. The time starts at the end of one\nread. The first read occurs when the pipeline starts.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"items": {
"description": "The position of the records in the reply, as a JSON pointer. For\nexample, `/data/machines` reads the array at `data.machines`. If you do\nnot set it, the input uses the full reply. An array gives one message\nfor each element. Any other value gives one message.",
"type": [
"string",
"null"
]
},
"max_batch": {
"description": "The maximum number of messages in one batch. The default is 1. The input\nputs only messages that are already read into a batch. It does not wait\nfor a batch to fill.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"timeout_seconds": {
"description": "The maximum time for one request, in s. The default is 30.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"url": {
"description": "The url to get, for example `https://erp.example.com/api/machines`.",
"type": "string"
}
},
"required": [
"url",
"interval_secs"
],
"title": "http_poll",
"type": "object"
},
"InduInputConfig": {
"description": "Reads live sensors and streams from Indu Cloud through `/api/v1/live/sse`,\nwith the API key of the connection.\n\nName sensors and streams with the ids that the platform uses. Do not use\nUUIDs. The input finds the names through `/api/v1` on the first read. If\nthe key cannot find or see a name, the input reports an error. It then\ntries again after a pause.\n\nEach reading is one message, for example\n`{\"kind\": \"sensor\", \"name\": \"press-3/temperature\", \"value\": 71.2, \"at\": …}`.\nThe message also contains the ids of the platform. When the connection\ndrops, the input connects again with backoff. If the input cannot read\nall readings, it reports an error.",
"properties": {
"backfill": {
"description": "Send the latest value of each series before the live readings. The\ndefault is true. Thus a restarted pipeline has a value for each series\nimmediately.",
"type": [
"boolean",
"null"
]
},
"connection": {
"description": "The name of the indu connection to read through. Declare the connection\nin the connections file.",
"type": "string",
"x-connection": "indu"
},
"max_batch": {
"description": "The maximum number of readings in one batch. The default is 1. The input\nputs only readings that are already received into a batch. It does not\nwait for more readings.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"sensors": {
"description": "The sensors to read, as `<device>/<sensor>`, for example\n`press-3/temperature`. Use the ids that the platform uses. kayak divides\nthe name at the first `/`.",
"items": {
"type": "string"
},
"type": "array"
},
"streams": {
"description": "The streams to read, by the name they were written under, for example\n`press-3/oee`. For a stream that the platform calculates, use its\ndisplay name.",
"items": {
"type": "string"
},
"type": "array"
}
},
"required": [
"connection"
],
"title": "indu",
"type": "object"
},
"InputConfig": {
"oneOf": [
{
"$ref": "#/$defs/DummyConfig",
"properties": {
"type": {
"const": "dummy",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/HttpInputConfig",
"properties": {
"type": {
"const": "http",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/KafkaConfig",
"properties": {
"type": {
"const": "kafka",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/NatsConfig",
"properties": {
"type": {
"const": "nats",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/PipelineConfig",
"properties": {
"type": {
"const": "pipeline",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/MqttConfig",
"properties": {
"type": {
"const": "mqtt",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/RedisConfig",
"properties": {
"type": {
"const": "redis",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/OpcuaConfig",
"properties": {
"type": {
"const": "opcua",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/InduInputConfig",
"properties": {
"type": {
"const": "indu",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/PostgresInputConfig",
"properties": {
"type": {
"const": "postgres",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/ClickhouseInputConfig",
"properties": {
"type": {
"const": "clickhouse",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"$ref": "#/$defs/HttpPollConfig",
"properties": {
"type": {
"const": "http_poll",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
}
],
"properties": {
"ack": {
"anyOf": [
{
"$ref": "#/$defs/AckMode"
},
{
"type": "null"
}
],
"description": "When the input acknowledges a message to its broker. The default is\n`on_receipt`. Only the `kafka` and `mqtt` inputs support `on_delivery`.\nThe `mqtt` input requires a `qos` of `at_least_once` or higher for it.\nOn all other inputs, `on_delivery` fails to build."
},
"buffer": {
"anyOf": [
{
"$ref": "#/$defs/BufferConfig"
},
{
"type": "null"
}
],
"description": "Collect messages from this input into batches before the transforms.\nUse a count (`static`), a time (`tumbling`) or the first of the two\n(`batch`). The buffer never sends an empty batch. Available on all\ninput types. This is not the `buffer` transform."
},
"envelope": {
"anyOf": [
{
"$ref": "#/$defs/EnvelopeConfig"
},
{
"type": "null"
}
],
"description": "Add metadata about the source of each message, for example the subject,\nthe topic or the partition. The \"metadata\" section lists the fields.\nAvailable on all input types. If you do not set it, the input sends each\nmessage without changes."
}
},
"type": "object"
},
"KafkaConfig": {
"description": "Consumes JSON messages from a kafka topic.\n\nThe input skips a payload that is not JSON and writes a warning to the log.\nThe consumer connects on the first read and joins a consumer group. Kafka\nkeeps the read position of the group between restarts.",
"properties": {
"connection": {
"description": "The name of the kafka connection to consume from. Declare the\nconnection in the connections file.",
"type": "string",
"x-connection": "kafka"
},
"group": {
"description": "The consumer group id. Kafka keeps one read position for each group.\nTwo pipelines in the same group divide the topic between them. Two\npipelines in different groups each get all messages.",
"type": "string"
},
"max_batch": {
"description": "The maximum number of messages in one batch. The default is 1.\n\nThe input puts only records that are already received into a batch. It\ndoes not wait for more records, so a high value does not add latency on\na quiet topic. Increase it when the consumer reads a backlog. Each batch\nhas a fixed cost in the pipeline and in each downstream pipeline.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"start_at": {
"anyOf": [
{
"$ref": "#/$defs/KafkaStartAt"
},
{
"type": "null"
}
],
"description": "The start position when the group has no committed position. `earliest`\nreads the topic from the start. `latest` reads only new messages. The\ndefault is `latest`."
},
"topic": {
"description": "The topic to consume from.",
"type": "string"
}
},
"required": [
"connection",
"topic",
"group"
],
"title": "kafka",
"type": "object"
},
"KafkaStartAt": {
"description": "The position at which a new consumer group starts to read.",
"enum": [
"earliest",
"latest"
],
"type": "string"
},
"MqttConfig": {
"description": "Subscribes to an mqtt topic and parses each message as JSON.\n\nThe topic can be a filter with the mqtt wildcards `+` and `#`. The input\nskips a payload that is not JSON and writes a warning to the log.\n\nThe input opens the connection on the first read. kayak makes the client id\nfrom the pipeline id and the topic. You cannot set the client id.",
"properties": {
"connection": {
"description": "The name of the mqtt connection to subscribe on. Declare the connection\nin the connections file.",
"type": "string",
"x-connection": "mqtt"
},
"max_batch": {
"description": "The maximum number of messages in one batch. The default is 1.\n\nThe input puts only messages that are already received into a batch.\nIt does not wait for more messages, so a high value does not add\nlatency on a quiet topic.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"qos": {
"anyOf": [
{
"$ref": "#/$defs/MqttQos"
},
{
"type": "null"
}
],
"description": "The quality of service for the subscription. The default is\n`at_most_once`. `ack: on_delivery` requires `at_least_once` or\n`exactly_once`."
},
"topic": {
"description": "The topic or topic filter to subscribe to.",
"type": "string"
}
},
"required": [
"connection",
"topic"
],
"title": "mqtt",
"type": "object"
},
"MqttQos": {
"description": "The mqtt quality of service for a subscribe or a publish.",
"oneOf": [
{
"const": "at_most_once",
"description": "QoS 0. The broker does not send a message again and there is no\nacknowledgement. The default.",
"type": "string"
},
{
"const": "at_least_once",
"description": "QoS 1. The broker sends a message again until it gets an\nacknowledgement, so a message can arrive more than one time. An input\nwith `ack: on_delivery` requires this level or higher.",
"type": "string"
},
{
"const": "exactly_once",
"description": "QoS 2. A four-part handshake makes sure of exactly one delivery. This\nlevel has the highest cost. Use `at_least_once` if a duplicate message\nis not a problem.",
"type": "string"
}
]
},
"NatsConfig": {
"description": "Subscribes to a nats subject and parses each message as JSON.\n\nThe input skips a payload that is not JSON and writes a warning to the log.\nThe input opens the connection on the first read.",
"properties": {
"connection": {
"description": "The name of the nats connection to subscribe on. Declare the connection\nin the connections file.",
"type": "string",
"x-connection": "nats"
},
"max_batch": {
"description": "The maximum number of messages in one batch. The default is 1.\n\nThe input puts only messages that are already received into a batch.\nIt does not wait for more messages, so a high value does not add\nlatency on a quiet subject.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"subject": {
"description": "The subject to subscribe to.",
"type": "string"
}
},
"required": [
"connection",
"subject"
],
"title": "nats",
"type": "object"
},
"OpcuaBrowseConfig": {
"description": "All variables under a node in the address space of the server. The input\nbrowses for them when the pipeline starts.\n\nThe input reads the address space only at the start. A tag that is added\nto the server later is read only after a restart. A tag that is removed\nstops without an error. Use a `nodes` list to name in the config file\nexactly which nodes the input reads. You can use `browse` and `nodes`\ntogether.",
"properties": {
"depth": {
"description": "The number of levels below the root to follow. The default is 3. The\nvalue 0 is not permitted. There is no value for \"all levels\", because\nthe address space of a plant server can have thousands of nodes.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"root": {
"description": "The id of the node to browse under, in the same notation as `node_id`.\nThis is usually a folder, for example `ns=2;s=Machine1`. The input\nsubscribes to each variable under it. It follows folders and objects\nbut does not subscribe to them.",
"type": "string"
}
},
"required": [
"root"
],
"title": "opcua browse",
"type": "object"
},
"OpcuaConfig": {
"description": "Subscribes to variables on an OPC UA server and sends one message for each\nchange of a value.\n\nThe input makes a subscription with one monitored item for each node. The\nserver sends a value when it changes. The input does not poll. A tag that\ndoes not change sends no messages. `publish_interval_ms` sets how\nfrequently the server can send. It does not set how frequently the server\nsamples.\n\nEach message is one reading. It contains the tag and the value:\n\n```json\n{\n \"node\": \"ns=2;s=Machine1.Temperature\",\n \"name\": \"temperature\",\n \"value\": 21.5,\n \"status\": \"Good\",\n \"source_timestamp\": \"2026-01-01T12:00:00.123Z\",\n \"server_timestamp\": \"2026-01-01T12:00:00.130Z\"\n}\n```\n\n`status` is the quality of the reading and is always present. A failed\nsensor sends a `Bad...` status with a `null` value. Use a `filter` to remove\nthese readings. `source_timestamp` is the time at which the device produced\nthe value. Use it to reduce or partition. The `received_at` field of the\nenvelope is the time at which kayak read the value.\n\nSet `nodes`, `browse` or both. One of them is required. The input subscribes\nto a node only one time, also when two settings name it.",
"properties": {
"browse": {
"anyOf": [
{
"$ref": "#/$defs/OpcuaBrowseConfig"
},
{
"type": "null"
}
],
"description": "A node to browse. The input subscribes to each variable under it."
},
"connection": {
"description": "The name of the opcua connection to subscribe on. Declare the\nconnection in the connections file.",
"type": "string",
"x-connection": "opcua"
},
"deadband": {
"description": "The smallest change of a value that the server reports, in the units of\nthe value. If you do not set it, the server reports each change.\n\nThe server applies the deadband, so it decreases network traffic. It\napplies only to numeric nodes. The server reports each change of a\nstring or a boolean.",
"format": "double",
"type": [
"number",
"null"
]
},
"max_batch": {
"description": "The maximum number of messages in one batch. The default is 1.\n\nOne publish from the server contains each node that changed in the\ninterval. With 200 tags at 1 Hz and the default, the pipeline handles\n200 batches each second. A higher value decreases this cost. The input\nputs only changes that are already received into a batch.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"nodes": {
"description": "The nodes to subscribe to, one entry for each node.",
"items": {
"$ref": "#/$defs/OpcuaNodeConfig"
},
"type": "array"
},
"publish_interval_ms": {
"description": "The interval at which the server can send a group of changes, in ms.\nThe default is 1000. This value sets the longest time that a change\nwaits.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"queue_size": {
"description": "The number of samples that the server keeps for one node between two\npublishes. The default is 1. With 1, the server sends only the latest\nvalue of a node that changes two times in one interval. Increase it,\ntogether with `sampling_interval_ms`, when you need each sample.",
"format": "uint32",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"sampling_interval_ms": {
"description": "The interval at which the server samples each node, in ms. If you do not\nset it, the server samples at the publish interval. Set a smaller value\nto get more readings between two publishes.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
}
},
"required": [
"connection"
],
"title": "opcua",
"type": "object"
},
"OpcuaNodeConfig": {
"description": "One node that an `opcua` input subscribes to, and the name for it in the\nmessages.",
"properties": {
"name": {
"description": "The name of the node in the messages. The default is the node id. Set\na readable name to use in a `group_by` or a column mapping.",
"type": [
"string",
"null"
]
},
"node_id": {
"description": "The id of the node, in OPC UA notation. Use `ns=2;s=Machine1.Temperature`\nfor a string identifier and `ns=2;i=1042` for a numeric identifier. Use\n`g=` for a GUID and `b=` for an opaque identifier. A node id with no\n`ns=` is in namespace 0, the namespace of the server.",
"type": "string"
}
},
"required": [
"node_id"
],
"title": "opcua node",
"type": "object"
},
"PipelineConfig": {
"description": "Reads the output of another pipeline. Use it to connect pipelines into a\ngraph.\n\nMany pipelines can read from the same upstream. Each of them gets all\nbatches. The upstream must exist when kayak creates this pipeline, so\ndeclare the upstream first in the config file.",
"properties": {
"upstream": {
"description": "The id of the pipeline to read from.",
"type": "string",
"x-pipeline-id": true
}
},
"required": [
"upstream"
],
"title": "pipeline",
"type": "object"
},
"PollMode": {
"description": "Whether a read returns all rows or only the new rows.",
"oneOf": [
{
"description": "Each read returns all rows, in one query. Use it for reference data,\nfor example a table of recipes for a `remember` transform. A snapshot\nhas no page limit. Use it only for a table that fits in memory.",
"properties": {
"type": {
"const": "snapshot",
"type": "string"
}
},
"required": [
"type"
],
"type": "object"
},
{
"description": "Each read returns only the rows with a `field` value above the highest\nvalue that the input already sent. The input reads pages in the order\nof the field. The value of the field must increase, for example an id\nor an `updated_at`. Put an index on the field. Without an index, each\nread scans the full table.",
"properties": {
"field": {
"description": "The column that the input follows. The watermark is the highest\nvalue that the input sent. Each read asks for the rows above the\nwatermark. The input does not read rows where the column is `null`.",
"type": "string"
},
"lag_secs": {
"description": "For a timestamp column: the time to stay behind the current time,\nin seconds. A later read gets the rows that are less than this time\nbefore `now()`. This gives late transactions time to commit. Do not\nuse it with a numeric column. The server refuses the query.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"start_from": {
"anyOf": [
{
"$ref": "#/$defs/StartFrom"
},
{
"type": "null"
}
],
"description": "Where the first read starts. With `newest`, the input reads only\nthe rows added after the pipeline started. With `oldest`, it reads\nall rows first and then follows the table. The default is `newest`."
},
"type": {
"const": "incremental",
"type": "string"
}
},
"required": [
"type",
"field"
],
"type": "object"
}
]
},
"PostgresInputConfig": {
"description": "Reads a postgres table, view or query at an interval and sends each row as\na message.\n\nThe input runs a query each `interval_secs`. In `snapshot` mode, it reads\nthe full relation. In `incremental` mode, it reads only the rows after the\nlast read, by a column that increases. Each row becomes one JSON object\nwith the column names as fields. Postgres makes the object with\n`row_to_json`. A timestamp is ISO 8601, a `numeric` keeps its digits, and\na `jsonb` column is a nested value:\n\n```json\n{\"id\": 42, \"sensor\": \"press-3\", \"value\": 21.5, \"recorded_at\": \"2026-01-01T12:00:00.123456+00:00\"}\n```\n\nAn incremental read is at-least-once across a restart. The input keeps the\nwatermark in memory, and after a restart it starts again from `start_from`.\nAn incremental read does not see a deleted row. Put an index on the column\nthat it follows, or each read scans the full table.",
"properties": {
"columns": {
"description": "The columns to read, in order. Leave it empty to read all columns. For\nan incremental input, the list must include `field`.",
"items": {
"type": "string"
},
"type": "array"
},
"connection": {
"description": "The name of the postgres connection to read through. Declare the\nconnection in the connections file.",
"type": "string",
"x-connection": "postgres"
},
"interval_secs": {
"description": "The time between two reads, in seconds. The time starts at the end of a\nread. Thus, two reads never overlap. The first read occurs when the\npipeline starts.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"max_batch": {
"description": "The maximum number of rows in one batch. The default is 1. The input\nputs rows that it already read into a batch. It does not wait for more\nrows to fill a batch.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"mode": {
"$ref": "#/$defs/PollMode",
"description": "Whether each read returns all rows (`snapshot`) or only the rows after\nthe last read (`incremental`)."
},
"page_size": {
"description": "The maximum number of rows that one query returns. The default is\n1000. When a page is full, the input reads the next page immediately.\nThe interval starts after a page that is not full. A `snapshot` ignores\nthis field and reads all rows in one query.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"query": {
"description": "A `SELECT` to read in place of a table. The input puts the query in a\nsubquery. It adds the cursor condition, the `ORDER BY` and the page\nlimit outside the subquery. Thus, the query needs no placeholder and no\n`ORDER BY`.\n\nWrite one statement with no semicolon at the end. You can use all SQL\nthat the server accepts in a subquery, for example `WITH`.",
"type": [
"string",
"null"
]
},
"table": {
"description": "The table or view to read, as `name` or `schema.name`. Give exactly one\nof `table` and `query`.",
"type": [
"string",
"null"
]
}
},
"required": [
"connection",
"interval_secs",
"mode"
],
"title": "postgres",
"type": "object"
},
"RedisConfig": {
"description": "Subscribes to a redis channel and parses each message as JSON.\n\nThe input skips a payload that is not JSON and writes a warning to the log.\nThe input opens the connection on the first read.\n\nThe input uses `SUBSCRIBE`, so the channel name must be exact. Patterns are\nnot supported. Redis pub/sub does not send a message again. When the input\nis not connected, it does not receive the messages that are published.",
"properties": {
"channel": {
"description": "The channel to subscribe to.",
"type": "string"
},
"connection": {
"description": "The name of the redis connection to subscribe on. Declare the\nconnection in the connections file.",
"type": "string",
"x-connection": "redis"
},
"max_batch": {
"description": "The maximum number of messages in one batch. The default is 1.\n\nThe input puts only messages that are already received into a batch.\nIt does not wait for more messages, so a high value does not add\nlatency on a quiet channel.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
}
},
"required": [
"connection",
"channel"
],
"title": "redis",
"type": "object"
},
"Secret": {
"description": "A text value that can refer to secrets with `${NAME}` references.\n\nThe value is an ordinary JSON string. When kayak builds the pipeline, it\nreplaces each `${NAME}` reference with the value from the secret store of\nthe server:\n\n```json\n{ \"type\": \"nats\", \"urls\": \"nats://app:${NATS_PASSWORD}@broker:4222\" }\n```\n\nThe config keeps only the reference, never the value. Thus you can commit\nthe file, and `GET /api/pipelines` does not show the secret.\n\nkayak uses a value with no `${...}` reference as it is.",
"type": "string"
},
"StartFrom": {
"description": "Where the first read of an incremental input starts.",
"oneOf": [
{
"const": "oldest",
"description": "From the start. The first read returns all rows, one page at a time.",
"type": "string"
},
{
"const": "newest",
"description": "From now. The first read finds the highest value of the field and\nreturns only the rows above it. This is the default.",
"type": "string"
}
]
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "Take a few messages from an input, without creating a pipeline.",
"properties": {
"input": {
"$ref": "#/$defs/InputConfig",
"description": "the input to read from, exactly as it would be written in a pipeline —\nincluding its `envelope`, so the metadata fields the messages will\nreally carry are in the sample too."
},
"max_messages": {
"description": "how many messages to take. Defaults to [`DEFAULT_MAX_MESSAGES`], capped\nat [`MAX_MESSAGES`].",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"timeout_ms": {
"description": "how long to wait for them, in milliseconds. Defaults to\n[`DEFAULT_TIMEOUT_MS`], capped at [`MAX_TIMEOUT_MS`]. The wait ends as\nsoon as enough messages have arrived.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
}
},
"required": [
"input"
],
"title": "sample request",
"type": "object"
}SampleResponse
What a sample found.
Note there is no separate "nothing arrived" arm: an empty messages is exactly that, and it is an ordinary answer rather than a failure. A stream nobody is publishing to is a real state of the world, and one the user often needs to be shown rather than protected from.
schema
json
{
"$defs": {
"SampleStage": {
"description": "Where a sample went wrong, when it did.",
"oneOf": [
{
"const": "build",
"description": "The input could not be built at all — a connection that isn't\nconfigured, a secret that doesn't resolve. The same failure creating\nthe pipeline would have given, which is the point of building the real\nthing.",
"type": "string"
},
{
"const": "read",
"description": "It was built, and reading from it failed — the broker refused the\nsubscription, the host is unreachable.",
"type": "string"
}
]
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "What a sample found.\n\nNote there is no separate \"nothing arrived\" arm: an empty `messages` is\nexactly that, and it is an ordinary answer rather than a failure. A stream\nnobody is publishing to is a real state of the world, and one the user\noften needs to be shown rather than protected from.",
"oneOf": [
{
"properties": {
"messages": {
"description": "the messages, exactly as the pipeline's first transform would have\nseen them.",
"items": true,
"type": "array"
},
"notes": {
"description": "what the sample did differently from the pipeline it is standing in\nfor — a throwaway consumer group, an ignored buffer. Empty when it\ndid nothing differently.",
"items": {
"type": "string"
},
"type": "array"
},
"outcome": {
"const": "sampled",
"type": "string"
},
"waited_ms": {
"description": "how long it waited, in milliseconds.",
"format": "uint64",
"minimum": 0,
"type": "integer"
}
},
"required": [
"outcome",
"messages",
"waited_ms"
],
"type": "object"
},
{
"properties": {
"message": {
"type": "string"
},
"outcome": {
"const": "failed",
"type": "string"
},
"stage": {
"$ref": "#/$defs/SampleStage"
}
},
"required": [
"outcome",
"stage",
"message"
],
"type": "object"
}
],
"title": "sample response"
}SaveConfigRequest
The body of POST /api/config/save. name is a file name with no path. kayak writes the file to the save directory of the server.
schema
json
{
"$defs": {
"ConfigFormat": {
"description": "The two ways a config file can be written.\n\nJSON is the default because it is what every existing file and every example\nin the repository uses; a file only gets read as YAML if it says so.",
"enum": [
"json",
"yaml"
],
"type": "string"
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "The body of `POST /api/config/save`. `name` is a file name with no path.\nkayak writes the file to the save directory of the server.",
"properties": {
"format": {
"anyOf": [
{
"$ref": "#/$defs/ConfigFormat"
},
{
"type": "null"
}
],
"default": null,
"description": "JSON or YAML. When it is absent, the extension of `name` sets the\nformat."
},
"name": {
"type": "string"
},
"overwrite": {
"default": true,
"description": "Whether the save can replace an existing file. The default is `true`.\nWith `false`, the save only makes new files. If a file with the name\nexists, the response is a 409 and kayak writes nothing.",
"type": "boolean"
}
},
"required": [
"name"
],
"title": "SaveConfigRequest",
"type": "object"
}SaveConfigResponse
The path of the file that the save wrote.
schema
json
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "The path of the file that the save wrote.",
"properties": {
"path": {
"type": "string"
}
},
"required": [
"path"
],
"title": "SaveConfigResponse",
"type": "object"
}SettingsDto
How the server started, and whether the running graph is the same as the config file.
schema
json
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "How the server started, and whether the running graph is the same as the\nconfig file.",
"properties": {
"config_file": {
"description": "The name of the config file of the server: the `--config` file, or the\nfile that a save made. When it is absent, there is no file yet. A save\nmakes one.",
"type": [
"string",
"null"
]
},
"save_directory": {
"default": "",
"description": "The directory that a save writes to. An empty value means that the\ndirectory is not known.",
"type": "string"
},
"unsaved_changes": {
"description": "True when the running graph is different from the last load or save.\nA change has an immediate effect on the server, but does not change the\nfile. A restart discards the unsaved changes.",
"type": "boolean"
}
},
"required": [
"unsaved_changes"
],
"title": "SettingsDto",
"type": "object"
}TokenLoginRequest
The body of POST /api/auth/token.
The token comes from the identity provider of the host application. kayak changes it into a session cookie. kayak does not store it, send it back or write it to the log.
schema
json
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "The body of `POST /api/auth/token`.\n\nThe token comes from the identity provider of the host application. kayak\nchanges it into a session cookie. kayak does not store it, send it back or\nwrite it to the log.",
"properties": {
"token": {
"type": "string"
}
},
"required": [
"token"
],
"title": "TokenLoginRequest",
"type": "object"
}UiEvent
One event in the /events stream.
schema
json
{
"$defs": {
"BatchPreview": {
"description": "A batch in the event feed: some of its messages as JSON text, and the\ncounts of the messages that the feed does not carry.\n\nThe feed carries a maximum of 100 messages for each batch. It cuts each\nmessage to 2048 bytes.",
"properties": {
"messages": {
"description": "The messages as compact JSON text. There are a maximum of 100. kayak\ncuts each one to 2048 bytes and adds `…` at the end of a cut message.",
"items": {
"type": "string"
},
"type": "array"
},
"skipped_messages": {
"default": 0,
"description": "The number of messages that passed this stage in passes that the feed\n**did not report**, since the last reported event.\n\nThe feed is a sample. To calculate the throughput, add this number to\n`total`.",
"format": "uint64",
"minimum": 0,
"type": "integer"
},
"total": {
"description": "The number of messages in the batch. It is larger than the length of\n`messages` when the batch has more than 100 messages.",
"format": "uint",
"minimum": 0,
"type": "integer"
}
},
"required": [
"messages",
"total"
],
"type": "object"
},
"EventPayload": {
"description": "The contents of an event: a batch that passed, or an error.",
"oneOf": [
{
"additionalProperties": false,
"properties": {
"batch": {
"$ref": "#/$defs/BatchPreview"
}
},
"required": [
"batch"
],
"type": "object"
},
{
"additionalProperties": false,
"description": "A failure at this stage, as text. The event does not carry the batch.",
"properties": {
"error": {
"type": "string"
}
},
"required": [
"error"
],
"type": "object"
}
]
},
"Stage": {
"description": "The stage of the run loop that an event comes from.",
"enum": [
"input",
"transform",
"output"
],
"type": "string"
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "One event in the `/events` stream.",
"properties": {
"component": {
"default": null,
"description": "The index of the component in the array of its stage in the config,\nfrom zero. For example, the second of two outputs is `1`.\n\nIt is `null` when the index is not known. Input events have no index,\nbecause kayak merges the inputs before the run loop gets a batch.",
"format": "uint",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"payload": {
"$ref": "#/$defs/EventPayload"
},
"pipeline_id": {
"type": "string"
},
"seq": {
"default": null,
"description": "The number of the pass through the run loop. A pass is one batch in, its\ntransforms, and its outputs. The count is per pipeline and starts at 1.\n\nIt is `null` for an event outside a pass. For example, an output that\nfailed to initialize before the loop started, or an input that failed\nwhile the loop waited.\n\nA gap shows missed passes. For example, a change from 8 to 12 means\nthat the client did not get three passes.",
"format": "uint64",
"minimum": 0,
"type": [
"integer",
"null"
]
},
"stage": {
"$ref": "#/$defs/Stage"
},
"ts": {
"default": 0,
"description": "The time of the event on the server clock, in milliseconds since the\nepoch. Zero means that the time is not known.",
"format": "uint64",
"minimum": 0,
"type": "integer"
}
},
"required": [
"pipeline_id",
"stage",
"payload"
],
"title": "UiEvent",
"type": "object"
}