{ "components": { "schemas": { "AckMode": { "description": "When an input acknowledges a message to its broker — see \"acknowledgement\nmodes\" in the guide for the reasoning and, importantly, its current scope.", "oneOf": [ { "const": "on_receipt", "description": "Acknowledge as soon as the message arrives, before any transform or\noutput has touched it. The default, and the behaviour every input has\nalways had — a crash between receipt and output can lose the message.", "type": "string" }, { "const": "on_delivery", "description": "Acknowledge once the message has left *this* pipeline: every output\nthis pipeline owns has returned, successfully or not, and every\ndownstream pipeline fed from here has accepted it into its inbox. A\nfailing output does not hold up the acknowledgement — see the\narchitecture notes on why that is the current line, not a permanent\none. Not yet propagated any further than this pipeline: a downstream\npipeline's own outputs are not waited on.", "type": "string" } ] }, "Aggregation": { "description": "One thing to compute over a group, and what to call it in the result.", "properties": { "as": { "description": "the field the emitted message carries this answer under. Two\naggregations may not share one, and none may collide with a `group_by`\nfield.", "type": "string" }, "field": { "description": "the field to aggregate. Required by every function except `count`, which\ncounts messages when it is left out.", "type": [ "string", "null" ], "x-message-field": true }, "function": { "$ref": "#/components/schemas/ReduceFnKind", "description": "how to combine the values" } }, "required": [ "function", "as" ], "title": "aggregation", "type": "object" }, "ApiError": { "description": "The error body every failing request comes back with.\n\nA Rust type rather than a hand-written schema because it has to stay in step\nwith what `AppError` actually serializes — `an_error_body_matches_the_documented_shape`\nin `tests/api.rs` is what says so.", "properties": { "error": { "description": "What went wrong, as one line. `anyhow`'s context chain is rendered into\nit, so the cause is in there as \"context: cause\" rather than nested.", "type": "string" } }, "required": [ "error" ], "title": "ApiError", "type": "object" }, "ArithmeticOperator": { "description": "What an [`Mapping::Arithmetic`] does with its two operands.", "oneOf": [ { "const": "add", "description": "left + right", "type": "string" }, { "const": "subtract", "description": "left − right", "type": "string" }, { "const": "multiply", "description": "left × right", "type": "string" }, { "const": "divide", "description": "left ÷ right. A literal zero on the right is refused when the pipeline\nis built; a *field* that turns out to be zero fails the batch.", "type": "string" } ] }, "AuthDto": { "description": "Who the caller is, and whether this server cares.\n\nThe frontend asks for this before it draws anything: it decides between the\nlogin page and the canvas, and between a canvas that can be edited and one\nthat can only be read.", "properties": { "authentication_required": { "description": "Whether this server checks credentials at all. `false` is a server\nstarted without a `--server-config`, or with one that sets\n`auth: {type: none}` — see [`crate::server_config`] for why that is the\ndefault.", "type": "boolean" }, "role": { "anyOf": [ { "$ref": "#/components/schemas/Role" }, { "type": "null" } ], "description": "What the caller may do. `None` means signed out — which is a different\nthing from [`Role::Read`], and worth keeping different: a reader may see\nthe graph, and a signed-out caller may not." }, "username": { "description": "The signed-in user, or `None` for a caller who presented nothing.", "type": [ "string", "null" ] } }, "required": [ "authentication_required" ], "title": "AuthDto", "type": "object" }, "BatchPreview": { "description": "A batch as the UI feed carries it: a few of its messages, already rendered\nand cut to size, plus the counts that say what was left out.\n\n**The truncation happens on the server**, which is the whole point of the\ntype. An earlier version sent `Arc` — the entire batch — and\nleft the browser to throw all but a hundred of them away, so a wide batch\nwas serialized whole, pushed across the wire whole and parsed whole before\nanything decided it wasn't wanted. At a kafka-shaped 50k messages a second\nthat measured 22 MB/s of JSON nobody ever read.", "properties": { "messages": { "description": "Compact JSON, at most [`MESSAGES_PER_BATCH`] of them, each cut to\n[`MAX_MESSAGE_BYTES`]. Compact rather than pretty because this is what a\ncollapsed row shows; expanding one re-parses it.", "items": { "type": "string" }, "type": "array" }, "skipped_messages": { "default": 0, "description": "Messages that passed this stage in passes the feed **did not report**,\ncounted since the last one it did — see `kayak::pipeline::UiThrottle`.\n\nIt exists so the throughput readout stays honest. The feed is sampled\nunder load, so counting only the batches that arrive would report a\nfraction of what the pipeline is really doing, and a card reading `40/s`\nunder a pipeline running at 40,000 says the wrong thing more loudly than\nno number at all would.", "format": "uint64", "minimum": 0, "type": "integer" }, "total": { "description": "How many messages the batch actually held. Larger than `messages` is\nlong whenever the batch was wider than the cap.", "format": "uint", "minimum": 0, "type": "integer" } }, "required": [ "messages", "total" ], "type": "object" }, "BucketContents": { "description": "What a bucket holds, for the UI's card.", "properties": { "entries": { "items": { "$ref": "#/components/schemas/BucketEntry" }, "type": "array" }, "keys": { "format": "uint", "minimum": 0, "type": "integer" }, "name": { "type": "string" }, "truncated": { "description": "Whether `entries` is short of `keys` because of the cap. A bucket can\nhold ten thousand keys and the card shows a page of them; saying so is\nwhat stops the card reading as the whole truth.", "type": "boolean" } }, "required": [ "name", "keys", "entries", "truncated" ], "title": "BucketContents", "type": "object" }, "BucketEntry": { "description": "One key's contents as the API reports them.", "properties": { "key": { "type": "string" }, "updated_at": { "type": "string" }, "values": { "additionalProperties": true, "type": "object" } }, "required": [ "key", "values", "updated_at" ], "type": "object" }, "BucketSummary": { "items": { "$ref": "#/components/schemas/BucketSummary" }, "title": "Array_of_BucketSummary", "type": "array" }, "BufferConfig": { "description": "How an input's messages are gathered into batches before the transforms see\nthem.\n\nAll three shapes are the same two limits with different halves left off — a\ncount, a time, or both, whichever is reached first. **A buffer never emits an\nempty batch**: the clock starts when the first message of a batch arrives,\nnot when the window was asked for, so an input that goes quiet emits nothing\nrather than a tick of nothing.\n\n`size` is a floor rather than a ceiling, the same rule a file output's\n`max_rows` follows: an arriving batch is never split, so an input already\nproducing batches of its own (`max_batch` on kafka and nats) can overshoot.", "oneOf": [ { "description": "Wait for a number of messages, however long that takes.", "properties": { "size": { "description": "how many messages to gather before the batch is handed on", "format": "uint", "minimum": 0, "type": "integer" }, "type": { "const": "static", "type": "string" } }, "required": [ "type", "size" ], "type": "object" }, { "description": "Wait for a length of time, however few messages that gathers — but at\nleast one. The window opens when the first message arrives.", "properties": { "type": { "const": "tumbling", "type": "string" }, "window_seconds": { "description": "how long to gather messages for, measured from the first one", "format": "uint", "minimum": 0, "type": "integer" } }, "required": [ "type", "window_seconds" ], "type": "object" }, { "description": "Both limits: whichever is reached first ends the batch. The usual\nchoice for a stream whose rate varies, since it bounds the batch size\nwhen the input is busy and the latency when it is quiet.", "properties": { "size": { "description": "how many messages end the batch immediately", "format": "uint", "minimum": 0, "type": "integer" }, "type": { "const": "batch", "type": "string" }, "window_seconds": { "description": "how long to wait for them, measured from the first message in the\nbatch", "format": "uint", "minimum": 0, "type": "integer" } }, "required": [ "type", "size", "window_seconds" ], "type": "object" } ] }, "BufferGateConfig": { "description": "A condition on a state bucket, as a release trigger for the `buffer`\ntransform.\n\nThe conditions are tested against the bucket entry rendered as an object —\nthe names `remember` wrote under are its fields — so `field` is a dotted\npath exactly as it is everywhere else, and several conditions mean *all of\nthem*, exactly as they do on `remember`'s `when`.\n\nNote what this is not: it is a gate on the whole buffer, not a test applied\nto each held message. When it opens, everything held is handed on.", "properties": { "bucket": { "description": "which bucket to watch. Defaults to the one this pipeline's `state`\nnames; a pipeline with no `state` of its own has to name it here.", "type": [ "string", "null" ] }, "conditions": { "description": "what has to be true of that key for the buffer to be released. All of\nthem, and at least one — a gate with no conditions would be a buffer\nthat releases on every write to the bucket.", "items": { "$ref": "#/components/schemas/Condition" }, "type": "array" }, "key": { "description": "which key in that bucket to read. A literal key, not a field path —\nthis is one gate for the whole buffer, so there is no message to take a\nkey from. Leave it out for the bucket-wide value, which is what\n`remember` writes when its pipeline's `state` has no `key`.", "type": [ "string", "null" ] } }, "required": [ "conditions" ], "title": "buffer gate", "type": "object" }, "BufferTransformConfig": { "description": "Holds messages back and hands them on when a *trigger* says to.\n\nThere are three triggers and they compose: a message count, a length of\ntime, and a condition on a state bucket. Any of them is enough on its own —\nwhichever comes first ends the wait, the same rule the input-level `batch`\nbuffer follows. A buffer with no trigger at all fails to build.\n\n`size` is the one that has always been here and it behaves exactly as it\ndid: messages are handed on in batches of exactly that many, as they fill.\nThe other two release **everything currently held** as a single batch,\nhowever much that is — which is the useful reading of \"the run is finished,\nsend what you have\".\n\nDistinct from the `buffer` option on an input: that one batches what an\ninput produces, before any transform has seen it. This one sits in the\nchain, so it batches what the transforms in front of it produced — after a\n`filter` has thinned the stream, or a `recall` has enriched it.", "properties": { "max_messages": { "description": "never hold more than this many messages: reaching it releases them all,\nwhatever the triggers say, and says so in the log once. Required unless\n`size` is set, because `size` is its own bound — a buffer waiting on a\ncondition that never comes true is otherwise a memory leak that grows\nat the rate of the stream.", "format": "uint", "minimum": 0, "type": [ "integer", "null" ] }, "seconds": { "description": "release everything held this many seconds after the *first* held\nmessage. The window opens when a message is held rather than when the\nlast batch went out, so this is a bound on how long a message waits and\nnot a cadence — an idle buffer holds nothing and no clock is running.", "format": "uint", "minimum": 0, "type": [ "integer", "null" ] }, "size": { "description": "hand messages on in batches of exactly this many, as they fill. On its\nown this is a buffer that only ever counts, and is what this transform\nhas always done.", "format": "uint", "minimum": 0, "type": [ "integer", "null" ] }, "until": { "anyOf": [ { "$ref": "#/components/schemas/BufferGateConfig" }, { "type": "null" } ], "description": "release everything held when a state bucket says so. This is the\ntrigger a *different* pipeline can pull: buckets are global, so one\npipeline can mark a run complete and this one hands on what it gathered\nwhile the run was going." } }, "title": "buffer", "type": "object" }, "CastType": { "description": "What a [`Mapping::Cast`] converts a value to.\n\nA closed set of *logical* shapes, and a deliberately smaller one than the\ncolumn mapping's `ColumnType` even though the two overlap. `integer` and\n`bigint` are one thing here, because JSON has one integer; `decimal` is\nabsent, because a `serde_json` number cannot hold one distinctly from a\nfloat and a cast that claimed to would be a lie. `json` means something else\nagain — in a column it is \"store whatever this is\", here it is \"this string\ncontains JSON, parse it\", which is the common case of a payload that arrived\ndouble-encoded.", "oneOf": [ { "const": "text", "description": "A string. A number or a boolean is written the way JSON writes it; an\nobject or an array is an error.", "type": "string" }, { "const": "integer", "description": "A whole number. A string is parsed; a number with a fractional part is\nan error rather than a rounding, since which way to round is not\nsomething a config file said.", "type": "string" }, { "const": "float", "description": "A number. A string is parsed.", "type": "string" }, { "const": "boolean", "description": "True or false. The strings `true`/`false` (in any case) and the numbers\n1/0 are accepted; nothing else is.", "type": "string" }, { "const": "timestamp", "description": "A timestamp, written out as RFC 3339. A string is parsed and\nre-rendered, so a mixture of offsets arrives downstream in one spelling;\na number is read as **seconds** since the epoch, fractions included —\nthe same reading the column mapping makes.", "type": "string" }, { "const": "date", "description": "A calendar date, written out as `2026-08-10`. A string may be a plain\ndate or a full timestamp, of which the date is taken.", "type": "string" }, { "const": "uuid", "description": "A UUID, lower-cased. Only a string in the canonical hyphenated form is\naccepted — this validates, it does not invent.", "type": "string" }, { "const": "json", "description": "The JSON a string contains, parsed. This is the one cast whose input\nmust be a string: it is for a payload that arrived encoded inside\nanother one.", "type": "string" } ] }, "ClickhouseConnection": { "description": "A ClickHouse server, as one user connects to it over its HTTP interface.\n\nThe same split [`PostgresConnection`] makes: the server, the database and\nthe user are the connection's; the *table* belongs to the output that writes\nit.\n\nThe HTTP interface rather than the native protocol because it is what every\nClickHouse deployment exposes — including ClickHouse Cloud, where 8443 is the\nonly port there is — and because it takes an insert as a body in a named\nformat, which is exactly the shape a batch of messages already has.", "properties": { "allow_http": { "description": "allow a plaintext `http://` url. Defaults to false: the credentials\nabove go with every insert, so sending them in the clear is a decision\nworth writing down. The local server in `docker-compose.yaml` is the\ncase that legitimately wants it.", "type": [ "boolean", "null" ] }, "database": { "description": "the database to write into. It has to exist already — an output creates\ntables, never databases.", "type": "string" }, "password": { "$ref": "#/components/schemas/Secret", "description": "that user's password. May reference secrets as `${NAME}` — see \"secrets\"\nin the readme, and prefer a reference to a literal here." }, "url": { "description": "url of the HTTP interface, e.g. `http://localhost:8123` for the server in\n`docker-compose.yaml`, or `https://:8443` for ClickHouse Cloud.", "type": "string" }, "user": { "description": "the user to connect as", "type": "string" } }, "required": [ "url", "database", "user", "password" ], "title": "clickhouse", "type": "object" }, "ClickhouseOutputConfig": { "description": "Inserts every batch into a ClickHouse table, one insert per batch.\n\n`columns` is spelled exactly as the postgres output's is — each entry names\na column, its type and the field to read, and `field` defaults to the\ncolumn's name. Without them the table gets a single column holding each\nmessage as JSON text.\n\nWhere it differs from postgres is what a created table is *sorted* by.\nClickHouse has no auto-increment column and no unique constraint, so there\nis no surrogate `id` to fall back on: `order_by` names the MergeTree sorting\nkey, and a table that names none is sorted by the `received_at` timestamp it\ngets for free. A sorting key does not deduplicate — naming one says how the\ntable is laid out and indexed, not that its rows are unique.\n\nThe table is created if it isn't there; set `create_table` to false for a\ntable someone else owns. Creation never *alters* an existing table.", "properties": { "columns": { "description": "which message field goes in which column. Leave it out to store each\nmessage whole, as JSON text, in a `payload` column.", "items": { "$ref": "#/components/schemas/ColumnMapping" }, "type": "array" }, "connection": { "description": "name of the clickhouse connection to insert through — see \"connections\"\nin the readme. The url, database and user live there; the table below is\nthis output's own.", "type": "string", "x-connection": "clickhouse" }, "create_table": { "description": "create the table on start if it does not exist. Defaults to true.", "type": [ "boolean", "null" ] }, "on_extra_fields": { "$ref": "#/components/schemas/ExtraFieldPolicy", "description": "what to do about a message carrying fields no column reads" }, "order_by": { "description": "the columns the created table is sorted by — MergeTree's sorting key, and\nits index. With none, the table gets a `received_at` timestamp of its own\nand is sorted by that. Named columns are made `NOT NULL`, since a\nnullable key is not something ClickHouse sorts by.", "items": { "type": "string" }, "type": "array" }, "table": { "description": "the table to insert into, created if it does not exist. Optionally\ndatabase-qualified (`analytics.readings`), which overrides the\nconnection's database; letters, digits and underscores only, since it\ncannot be sent as a query parameter.", "type": "string" } }, "required": [ "connection", "table" ], "title": "clickhouse", "type": "object" }, "ColumnMapping": { "description": "One message field mapped onto one column.\n\n`field` defaults to `name`, so a message that already uses the column names\nneeds nothing but the name and the type. It is a dotted path like every\nother field reference in kayak, so `_meta.subject` reaches whatever the\ninput's envelope attached and a literal key containing dots still wins.", "properties": { "field": { "description": "the field to read, as a dotted path. Defaults to the column's name.", "type": [ "string", "null" ], "x-message-field": true }, "message": { "description": "store the whole message in this column instead of one of its fields.\nOnly for a `json` column, and not together with `field`.", "type": "boolean" }, "name": { "description": "the column's name in the table. Letters, digits and underscores only,\nsince it cannot be sent as a query parameter.", "type": "string" }, "nullable": { "description": "whether the column accepts `NULL`. Defaults to true; `false` makes the\ncreated column `NOT NULL` and makes a missing field an error.", "type": [ "boolean", "null" ] }, "on_missing": { "anyOf": [ { "$ref": "#/components/schemas/MissingColumnPolicy" }, { "type": "null" } ], "description": "what to do about a message that doesn't carry the field. Defaults to\n`null`, or to `error` for a column that is not nullable." }, "type": { "$ref": "#/components/schemas/ColumnType", "description": "what the column holds. Values are checked against it rather than\ncoerced into it." } }, "required": [ "name", "type" ], "title": "column", "type": "object" }, "ColumnType": { "description": "The type a column holds, named the way the *config* thinks about it rather\nthan the way any one server spells it.\n\nValues are checked against this before they are sent: a string `\"12.5\"` into\na `float` column is an error, not a coercion. Guessing is the failure mode\nnobody sees, and a type that can be coerced from anything makes the mapping\nworth nothing.", "oneOf": [ { "const": "text", "description": "A string of any length. Only a JSON string is accepted.", "type": "string" }, { "const": "integer", "description": "A 32-bit whole number. A JSON number with a fractional part, or one\noutside the range, is an error rather than a rounding.", "type": "string" }, { "const": "bigint", "description": "A 64-bit whole number.", "type": "string" }, { "const": "float", "description": "A double-precision floating point number.", "type": "string" }, { "const": "decimal", "description": "An exact decimal. The digits are carried across as they were written, so\nnothing is lost to a binary float on the way.", "type": "string" }, { "const": "boolean", "description": "True or false. Only a JSON boolean is accepted.", "type": "string" }, { "const": "timestamp", "description": "A date and time with a time zone. A JSON string is parsed by the server\n(ISO 8601 / RFC 3339); a JSON number is read as **seconds** since the\nepoch, fractions included.", "type": "string" }, { "const": "date", "description": "A calendar date, as a JSON string (`2026-08-10`).", "type": "string" }, { "const": "uuid", "description": "A UUID, as a JSON string.", "type": "string" }, { "const": "json", "description": "Any JSON value at all, stored as JSON.", "type": "string" } ] }, "ComponentDoc": { "description": "One component: everything `/docs` shows about it.", "properties": { "description": { "description": "The config struct's doc comment, if it has one.", "type": [ "string", "null" ] }, "family": { "$ref": "#/components/schemas/Family" }, "fields": { "items": { "$ref": "#/components/schemas/FieldDoc" }, "type": "array" }, "kind": { "description": "The `type` tag that selects this component in a config file.", "type": "string" }, "metadata": { "description": "What this input attaches to a message when its `envelope` is set —\nempty for every family but [`Family::Input`], and declared in\n[`crate::metadata`] rather than reflected, since a schema cannot know\nwhat a nats subscription knows.", "items": { "$ref": "#/components/schemas/MetaFieldDoc" }, "type": "array" }, "variants": { "description": "Empty for all but the enum-shaped components; when it isn't, the fields\nlive on the variants instead.", "items": { "$ref": "#/components/schemas/VariantDoc" }, "type": "array" } }, "required": [ "kind", "family", "fields", "variants" ], "title": "ComponentDoc", "type": "object" }, "ConcatPart": { "description": "One piece of a [`Mapping::Concat`].", "oneOf": [ { "description": "A value read out of the message. A string is taken as it is; a number or\na boolean is written the way JSON writes it. An object or an array is an\nerror — there is no one right way to flatten one into a key.", "properties": { "field": { "description": "the field to read", "type": "string" }, "type": { "const": "field", "type": "string" } }, "required": [ "type", "field" ], "type": "object" }, { "description": "Literal text — the separator, a prefix, a suffix.", "properties": { "type": { "const": "value", "type": "string" }, "value": { "description": "the text", "type": "string" } }, "required": [ "type", "value" ], "type": "object" } ] }, "Condition": { "description": "One test a message either passes or doesn't.\n\nThe same comparisons the `filter` transform makes, spelled as a tagged union\nso that a *list* of them can be configured and rendered as a form. Several\nconditions are read as \"all of these\" — there is no `or` and no nesting,\nbecause the moment either exists this is an expression language with a\nsyntax to design, and everything so far has been reachable without one.", "oneOf": [ { "description": "Compares a field to a number. A message whose field is missing or isn't\na number does not match.", "properties": { "field": { "description": "the field to test — a dotted path, like anywhere else", "type": "string" }, "operator": { "$ref": "#/components/schemas/NumericFilterOperatorKind" }, "type": { "const": "numeric", "type": "string" }, "value": { "format": "double", "type": "number" } }, "required": [ "type", "field", "operator", "value" ], "type": "object" }, { "description": "Compares a field to a string, the same way.", "properties": { "field": { "description": "the field to test — a dotted path, like anywhere else", "type": "string" }, "operator": { "$ref": "#/components/schemas/StringFilterOperatorKind" }, "type": { "const": "string", "type": "string" }, "value": { "type": "string" } }, "required": [ "type", "field", "operator", "value" ], "type": "object" } ] }, "Config": { "description": "One pipeline: every input is merged into one stream, that stream runs\nthrough the transform chain in order, and each resulting batch goes to every\noutput.", "properties": { "id": { "type": [ "string", "null" ] }, "inputs": { "description": "at least one. Batches arrive interleaved in the order the inputs produce\nthem; there is no ordering between two different inputs.", "items": { "$ref": "#/components/schemas/InputConfig" }, "type": "array" }, "outputs": { "default": [], "description": "may be omitted — a pipeline that only feeds downstream pipelines needs no\noutput of its own.", "items": { "$ref": "#/components/schemas/OutputConfig" }, "type": "array" }, "state": { "anyOf": [ { "$ref": "#/components/schemas/PipelineState" }, { "type": "null" } ], "description": "the state bucket this pipeline remembers things in, and what its\nmessages are keyed by. Only needed by a pipeline with a `remember` or\n`recall` transform; those fail to build without it." }, "transforms": { "default": [], "description": "may be omitted — a pipeline that only moves messages needs no transform.", "items": { "$ref": "#/components/schemas/TransformConfig" }, "type": "array" } }, "required": [ "inputs" ], "title": "Config", "type": "object" }, "ConfigFormat": { "description": "The two ways a config file can be written.\n\nJSON is the default because it is what every existing file and every example\nin the repository uses; a file only gets read as YAML if it says so.", "enum": [ "json", "yaml" ], "type": "string" }, "ConnectionKind": { "description": "Every kind of system a connection can describe.\n\nTagged the same way the component enums are, so a connection reads like the\ncomponents that use it. One kind serves both directions: a `kafka`\nconnection is what a kafka input consumes from *and* what a kafka output\npublishes to.", "oneOf": [ { "$ref": "#/components/schemas/KafkaConnection", "properties": { "type": { "const": "kafka", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/NatsConnection", "properties": { "type": { "const": "nats", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/PostgresConnection", "properties": { "type": { "const": "postgres", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/ClickhouseConnection", "properties": { "type": { "const": "clickhouse", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/FileConnection", "properties": { "type": { "const": "file", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/S3Connection", "properties": { "type": { "const": "s3", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/MqttConnection", "properties": { "type": { "const": "mqtt", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/RedisConnection", "properties": { "type": { "const": "redis", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/OpcuaConnection", "properties": { "type": { "const": "opcua", "type": "string" } }, "required": [ "type" ], "type": "object" } ] }, "Connections": { "additionalProperties": { "$ref": "#/components/schemas/ConnectionKind" }, "description": "Everything in the connections file, by name.\n\nA `BTreeMap` rather than a list of `{id, ...}` objects: the name is the\nidentity, duplicates are impossible to express, and iteration is in name\norder — which is what makes the file deterministic to write, the same\nproperty the config file depends on.", "title": "Connections", "type": "object" }, "CreateConnectionRequest": { "description": "What `POST /api/connections` takes: a name, and the connection itself\nflattened alongside it.\n\nThe name is a field here rather than a path segment because it is part of\nwhat is being created, and because the body then reads exactly like one\nentry of the file it will be written to.", "oneOf": [ { "$ref": "#/components/schemas/KafkaConnection", "properties": { "type": { "const": "kafka", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/NatsConnection", "properties": { "type": { "const": "nats", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/PostgresConnection", "properties": { "type": { "const": "postgres", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/ClickhouseConnection", "properties": { "type": { "const": "clickhouse", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/FileConnection", "properties": { "type": { "const": "file", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/S3Connection", "properties": { "type": { "const": "s3", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/MqttConnection", "properties": { "type": { "const": "mqtt", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/RedisConnection", "properties": { "type": { "const": "redis", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/OpcuaConnection", "properties": { "type": { "const": "opcua", "type": "string" } }, "required": [ "type" ], "type": "object" } ], "properties": { "id": { "type": "string" } }, "required": [ "id" ], "title": "CreateConnectionRequest", "type": "object" }, "DryRunRequest": { "description": "What `POST /api/scripts/dry-run` takes.\n\nThe endpoint exists because a script is the one component whose\nconfiguration can be *wrong in a way the config's shape cannot express*. For\nevery other component, a config that deserializes and builds is a component\nthat does what it says; for this one, the interesting mistakes are all\ninside a string. Without somewhere to run it, the only way to find out is to\ncreate a pipeline and watch its card — which for the HTTP API means creating\na *running* pipeline you then have to tear down.\n\nBoth the transform and this run through the same\n`kayak::transforms::script::runner`, configured identically. That is not a\nconvenience: a dry run whose agreement with production is a matter of luck\nis worse than no dry run, because it is trusted.", "properties": { "max_operations": { "description": "the operation budget for this run. Left out, the same default a\ntransform gets.", "format": "uint64", "minimum": 0, "type": [ "integer", "null" ] }, "messages": { "default": [], "description": "the messages to run it over — one batch. An empty list is allowed and is\nhow a script is checked for compiling without inventing data for it.", "items": true, "type": "array" }, "scope": { "$ref": "#/components/schemas/ScriptScope", "default": "message", "description": "whether the script sees one message at a time or the whole batch" }, "source": { "$ref": "#/components/schemas/ScriptSource", "description": "the script to run, inline or by reference, exactly as a transform\ndeclares it" }, "state": { "additionalProperties": { "additionalProperties": true, "type": "object" }, "description": "state to seed the run's bucket with, keyed by the key a script would\n`recall` it under.\n\nA dry run **never touches a live bucket**: it gets a private one, seeded\nfrom here and thrown away afterwards. Reading production state would\nmake a dry run's answer depend on what the server happened to be doing,\nand writing it would give a \"dry\" run side effects — the second being\nthe one that would be found out late and badly.", "type": "object" } }, "required": [ "source" ], "title": "DryRunRequest", "type": "object" }, "DryRunResponse": { "description": "What came back from a dry run.\n\nA script that does not compile comes back **200 with a `failed` outcome**,\nnot a 400. The request was well formed and the server answered it\ncompletely; \"this script has a bug on line 3\" is the answer, not a failure\nto produce one. A 400 would conflate a malformed request with a working\nendpoint reporting what it was asked to find out, and a client would have to\ntell them apart by reading the body anyway.", "oneOf": [ { "description": "The script ran. Note this includes a script that emitted nothing —\ndropping every message is a working filter, not a failure.", "properties": { "batches": { "description": "the batches the script emitted, in order. In `message` scope this is\nat most one; in `batch` scope it is however many the script asked\nfor.", "items": { "items": true, "type": "array" }, "type": "array" }, "outcome": { "const": "emitted", "type": "string" }, "state": { "additionalProperties": { "additionalProperties": true, "type": "object" }, "description": "what the run's private bucket holds afterwards — what the script\nwould have remembered. Discarded when the response is sent.", "type": "object" }, "warnings": { "description": "distinct texts the script passed to `warn()`", "items": { "type": "string" }, "type": "array" } }, "required": [ "outcome", "batches" ], "type": "object" }, { "description": "The script did not compile, or a run of it failed.", "properties": { "column": { "description": "one-based, beside `line` and absent for the same reasons", "format": "uint", "minimum": 0, "type": [ "integer", "null" ] }, "line": { "description": "one-based, as an editor counts. Absent when the failure belongs to\nthe run rather than to a line — an exhausted budget, or an error\nraised by a host function.", "format": "uint", "minimum": 0, "type": [ "integer", "null" ] }, "message": { "description": "what went wrong, in rhai's words and without the position appended —\nthe position is beside it, as a number a client can use", "type": "string" }, "outcome": { "const": "failed", "type": "string" }, "stage": { "$ref": "#/components/schemas/DryRunStage", "description": "whether this stopped the script compiling or stopped one run of it.\nThe first would refuse to start a pipeline; the second would fail a\nbatch on one that was already running." } }, "required": [ "outcome", "stage", "message" ], "type": "object" } ], "title": "DryRunResponse" }, "DryRunStage": { "description": "Which half of a script's life a failure belongs to.", "oneOf": [ { "const": "compile", "description": "The script does not parse. A pipeline with this script refuses to start.", "type": "string" }, { "const": "runtime", "description": "The script parsed and this run of it failed. A pipeline with this script\nstarts, and fails the batches that hit it.", "type": "string" } ] }, "DummyConfig": { "description": "Emits one generated message on a fixed interval — a heartbeat for testing a\npipeline without a real source attached.\n\nEvery message carries a `value` and the `current_time` it was emitted at.\nWhat the `value` holds is the `payload` field's business: a number sampled\nfrom a sine wave, so a chart of it has a shape, or a random sentence, so a\ntext transform has something to chew on.", "properties": { "amplitude": { "description": "peak of the sine wave — it swings between `-amplitude` and `+amplitude`.\nNumeric payloads only; defaults to 1.", "format": "double", "type": [ "number", "null" ] }, "duration": { "description": "seconds between messages", "format": "uint64", "minimum": 0, "type": "integer" }, "payload": { "anyOf": [ { "$ref": "#/components/schemas/DummyPayload" }, { "type": "null" } ], "description": "what each message's `value` holds: a `number` sampled from a sine wave,\nor a random sentence as `text`. Defaults to `number`." }, "period": { "description": "seconds for one full turn of the sine wave. Numeric payloads only;\ndefaults to 60. Sampling is by wall clock rather than by message count,\nso the wave keeps its period whatever `duration` is.", "format": "double", "type": [ "number", "null" ] } }, "required": [ "duration" ], "title": "dummy", "type": "object" }, "DummyPayload": { "description": "What a dummy input puts in each message's `value`.", "oneOf": [ { "const": "number", "description": "a number sampled from a sine wave", "type": "string" }, { "const": "text", "description": "a random sentence", "type": "string" } ] }, "EdgeLayout": { "description": "One edge's routing, where the automatic answer wasn't the readable one.\n\nAn edge between two cards runs out of one, along a channel between them, and\ninto the other. `offset` moves that channel off the half-way line — pulled\ntowards one card or the other — which is how a route that the automatic\nseparation put somewhere unhelpful gets moved. The two ports say where on\ntheir cards the edge attaches.\n\n`None` is not \"zero\": it means nobody has touched this channel, so the canvas\nis free to place it — and it does, spreading edges that would otherwise lie\non top of each other onto lines of their own. A stored `Some(0.0)` is a\ndeliberate \"on the half-way line, whatever else is there\", which is why this\nis an option rather than a number with a default.\n\nThe offset is *relative* rather than an absolute coordinate, because cards\nmove: storing \"20 above the middle\" keeps the adjustment meaningful after\neither end is dragged, where storing a y would eventually put the channel\noutside the gap it was meant to sit in.\n\nListed rather than keyed by a joined id, so nothing has to be escaped and no\nid character is special.", "properties": { "from": { "type": "string" }, "from_port": { "anyOf": [ { "$ref": "#/components/schemas/PortLayout" }, { "type": "null" } ] }, "offset": { "format": "double", "type": [ "number", "null" ] }, "to": { "type": "string" }, "to_port": { "anyOf": [ { "$ref": "#/components/schemas/PortLayout" }, { "type": "null" } ] } }, "required": [ "from", "to" ], "type": "object" }, "EnvelopeConfig": { "description": "Whether — and how — an input attaches metadata about where a message came\nfrom.\n\nThe metadata itself is documented per input under \"metadata\" on this page:\nthe subject a nats message arrived on, the topic, partition and offset of a\nkafka record, and so on, plus the pipeline and input kind that read it. It\nis attached **in band**, as ordinary fields on the message, so every\ntransform can filter, group and aggregate on it exactly as it does on the\npayload's own fields — `\"group_by\": [\"_meta.subject\"]` needs nothing new.\n\nLeaving this out is the default and means what it always meant: the message\nis passed on exactly as it arrived. Attaching metadata changes the shape of\nevery message from this input, which is not something to do to a running\nconfig without being asked.", "oneOf": [ { "description": "Add the metadata as one more field on the message. The payload's own\nfields stay exactly where they were, so nothing downstream has to\nchange.\n\nOnly works on a payload that is a JSON *object*: a message that is a\nbare number or string has nowhere to put the field, and is skipped with\na warning rather than taking the pipeline down. Use `wrap` for those.", "properties": { "meta": { "description": "the field the metadata object is written to. Defaults to `_meta`.", "type": [ "string", "null" ] }, "type": { "const": "merge", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "description": "Put the whole payload under a field of its own, beside the metadata —\n`{\"value\": , \"_meta\": {…}}`.\n\nWorks whatever the payload is, which is what a source of bare readings\n(a `1`, a `\"recipe-a\"`) needs. The cost is that every field reference\ndownstream now goes through the payload field: `value.temperature`\nrather than `temperature`.", "properties": { "meta": { "description": "the field the metadata object is written to. Defaults to `_meta`.", "type": [ "string", "null" ] }, "payload": { "description": "the field the original payload is written to. Defaults to `value`.", "type": [ "string", "null" ] }, "type": { "const": "wrap", "type": "string" } }, "required": [ "type" ], "type": "object" } ] }, "ErrorSignature": { "description": "One distinct failure, and how it has behaved over time.\n\n**Aggregated rather than logged**, which is the difference between a useful\nmorning readout and two million rows to scroll. A pipeline whose broker went\ndown at 02:14 and stayed down is one of these saying so, with a count — and\nthat is both cheaper to keep and easier to read than the log it replaces.\n\nIdentity is (`stage`, `component`, `message`): the same text from the second\nof two outputs is a different fact from the first one's, which is the same\nrule the run loop's failure budget already uses.", "properties": { "component": { "description": "Which component of that stage, indexed into its array in the config.\n`None` where the run loop doesn't know — an input failure, since inputs\nare merged before the loop sees them.", "format": "uint", "minimum": 0, "type": [ "integer", "null" ] }, "count": { "description": "How many times it has happened, including the repeats the log\nsuppressed. See [`HistoryBucket::errors`] — same accounting.", "format": "uint64", "minimum": 0, "type": "integer" }, "first_seen": { "description": "When it was first seen, in milliseconds since the epoch. This is the\nnumber the morning question is actually about.", "format": "uint64", "minimum": 0, "type": "integer" }, "last_seen": { "description": "When it was last seen. Equal to `first_seen` for a one-off; far from it\nfor something still broken, which is how the two are told apart.", "format": "uint64", "minimum": 0, "type": "integer" }, "message": { "description": "The failure's text, as the log line would have shown it, cut to\n[`crate::MAX_MESSAGE_BYTES`]. Cut rather than kept whole for the reason\nthe feed's messages are: an error with a payload embedded in it can be\narbitrarily long, and this is a store that promises to be bounded.", "type": "string" }, "stage": { "$ref": "#/components/schemas/Stage", "description": "Where in the pass it failed." } }, "required": [ "stage", "message", "first_seen", "last_seen", "count" ], "type": "object" }, "EventPayload": { "description": "What a run loop is reporting: a batch that passed through, or something that\nwent wrong while handling one.", "oneOf": [ { "additionalProperties": false, "properties": { "batch": { "$ref": "#/components/schemas/BatchPreview" } }, "required": [ "batch" ], "type": "object" }, { "additionalProperties": false, "description": "A failure at this stage. The batch that caused it is not carried: a\ntransform that failed has no output to show, and the input that did\narrive was already reported by its own event.", "properties": { "error": { "type": "string" } }, "required": [ "error" ], "type": "object" } ] }, "ExtraFieldPolicy": { "description": "What to do about a message carrying fields no column reads.", "oneOf": [ { "const": "ignore", "description": "Write the columns that are mapped and let the rest go. The default —\nmapping a subset of a wide message is the ordinary reason to map at all.", "type": "string" }, { "const": "error", "description": "Fail the pipeline. For a stream whose shape is supposed to be fixed,\nwhere a new field appearing is news rather than noise.", "type": "string" } ] }, "FailurePhase": { "description": "Where a chain went wrong, when it did.", "oneOf": [ { "const": "build", "description": "The transform could not be built — the same failure creating the\npipeline would have given, which is most of the value of building the\nreal thing.", "type": "string" }, { "const": "apply", "description": "It was built and failed on a message. A running pipeline would drop\nthat batch and carry on; a dry run stops, because what happened at the\npoint of failure is the thing being asked about.", "type": "string" } ] }, "Family": { "description": "Which plugin point a component plugs into. Also the grouping the docs\nsidebar uses, which is why it's ordered the way a pipeline reads.\n\nA connection isn't a stage of a pipeline, but it is configured the same way\n— a tagged struct with doc-commented fields — so it documents itself and\ngenerates its form through exactly this machinery.", "enum": [ "input", "transform", "output", "connection" ], "type": "string" }, "FieldDoc": { "description": "One configurable field of a component.", "properties": { "description": { "description": "The field's doc comment, if it has one.", "type": [ "string", "null" ] }, "field_type": { "$ref": "#/components/schemas/FieldType", "description": "The same thing in a form a UI can dispatch on." }, "name": { "description": "The wire name — what actually goes in the JSON.", "type": "string" }, "required": { "description": "Required fields have to appear in the JSON; optional ones may be omitted.", "type": "boolean" }, "type_name": { "description": "A rendered type, e.g. `string`, `integer`, or `sum | avg | min | max`\nfor a field that only accepts certain values.", "type": "string" } }, "required": [ "name", "type_name", "field_type", "required" ], "type": "object" }, "FieldType": { "description": "What a field actually accepts, as something a form can be built from.\n\n[`FieldDoc::type_name`] is the same information rendered for a human to\nread; this is the machine-readable half, and it is what the \"add pipeline\"\nmodal picks a widget and a validation rule from. Keeping it here rather\nthan in the frontend is the same bargain as the rest of this module: the\nconfig schema stays the single source of truth, so a new field gets a\nworking form control without anyone editing the UI.", "oneOf": [ { "enum": [ "text", "integer", "number", "boolean" ], "type": "string" }, { "additionalProperties": false, "description": "A closed set of accepted values — rendered as a dropdown, not a box.", "properties": { "enum": { "items": { "type": "string" }, "type": "array" } }, "required": [ "enum" ], "type": "object" }, { "const": "pipeline_id", "description": "The id of another pipeline. A string on the wire, but the set of valid\nanswers is the graph the server is currently running, which only the UI\nknows — so it renders as a dropdown of the pipelines that exist rather\nthan as a box to retype an id into.", "type": "string" }, { "additionalProperties": false, "description": "The name of a configured connection, of the kind carried here (`kafka`,\n`nats`, ...). Like [`FieldType::PipelineId`] it is a string on the wire\nwhose valid answers are server state, so it renders as a dropdown — of\nthe connections of *that kind*, since a nats connection is no use to a\nkafka input.", "properties": { "connection": { "type": "string" } }, "required": [ "connection" ], "type": "object" }, { "const": "message_field", "description": "The name of a field *in the messages* — a column's `field`, a filter's\ncomparison, an aggregation's source. A string on the wire like\n[`FieldType::PipelineId`], and a separate type for the same reason:\nwhat the control should be is not derivable from the wire type.\n\nUnlike the two dropdowns above this one is a **box with suggestions**,\nnever a closed list. The suggestions come from a sample of real\nmessages ([`crate::schema::infer`]), and a sample is a handful of\nmessages rather than a schema — a stream that carries `error_code` only\nwhen something breaks would otherwise have no way to say so.", "type": "string" }, { "additionalProperties": false, "description": "A value that is one of several shapes, tagged by one of its own\nproperties — an input's `buffer`, which is `{\"type\": \"static\", \"size\":\n10}` or `{\"type\": \"tumbling\", \"window_seconds\": 30}`.\n\nThis is the field-level twin of [`ComponentDoc::variants`], and it is\nwhat makes a form conditional: which fields a value has depends on which\nvariant was picked, so the tag is chosen first and the rest of the form\nfollows from it.", "properties": { "union": { "$ref": "#/components/schemas/UnionDoc" } }, "required": [ "union" ], "type": "object" }, { "additionalProperties": false, "description": "A value that is an object with a fixed set of fields of its own — a file\noutput's `rotate`. Not a choice, just a level of nesting, so the form\nrenders its fields inline.", "properties": { "object": { "items": { "$ref": "#/components/schemas/FieldDoc" }, "type": "array" } }, "required": [ "object" ], "type": "object" }, { "additionalProperties": false, "description": "A value that is a list of some other type — a reducer's `aggregations`.\nUnlike every other field type this one has no fixed number of boxes, so\nthe form renders rows that can be added and taken away, each of them\nwhatever the element type asks for.\n\nThe element is carried as a whole [`FieldDoc`] because that is what a\ncontrol is chosen from, and its [`FieldDoc::name`] is empty: a list\nelement has no name of its own, it has a position, which only the form\nrendering it knows. It is always required — a row that is there is a\nvalue that will be sent.", "properties": { "list": { "$ref": "#/components/schemas/FieldDoc" } }, "required": [ "list" ], "type": "object" }, { "additionalProperties": false, "description": "Source code in some language, carried as a string — a `script`\ntransform's `code`. The language is the payload (`rhai`), because a form\ncannot syntax-highlight what it does not know the name of.\n\nA string on the wire like [`FieldType::Text`], and a separate type for\nthe reason [`FieldType::PipelineId`] is: what the *control* should be is\nnot derivable from the wire type. A one-line box is the wrong shape for\ntwenty lines of code, and tab is a character rather than a way out of\nthe field.", "properties": { "script": { "type": "string" } }, "required": [ "script" ], "type": "object" }, { "const": "json", "description": "Something with a shape of its own that is none of the above — a union\ntagged in a spelling this module doesn't read, say. There is no general\nwidget for those, so the form takes them as literal JSON.", "type": "string" } ] }, "FileConnection": { "description": "A directory on the server's filesystem that file outputs write under.\n\nThe odd one out among the kinds: there is no host, no credentials, nothing\nto authenticate against. It earns its place as a connection anyway because\nit holds the same thing the others do — *what the system is*, as against\nwhat one pipeline wants from it. A file output names a `path` relative to\nthis root exactly as a kafka output names a topic on those brokers, and the\nobject-store connection that replaces it later swaps the root for a bucket\nwithout any component changing.\n\nThe root is **not** a boundary on its own. It arrives from `POST\n/api/connections` like any other connection, so a browser could name `/`\nhere; what actually confines writes is the server's `--data-dir`, which this\nroot has to resolve under. See `Root::resolve` in the root crate.", "properties": { "root": { "description": "directory that file outputs write under, e.g. `./out/events`. Created if\nit does not exist, and it must resolve inside the server's `--data-dir`\n— a server started without that flag has file output turned off.", "type": "string" } }, "required": [ "root" ], "title": "file", "type": "object" }, "FileFormat": { "description": "How the messages in a file are laid out.\n\nBoth are JSON — the difference is whether the file is one document or one\ndocument per line. `ndjson` is the one to want for anything that streams:\nthe file is valid after every batch, so a run that is still going (or that\ndied) is still readable, and every tool that eats logs eats it.", "oneOf": [ { "const": "ndjson", "description": "one JSON message per line, appended as it arrives", "type": "string" }, { "const": "json_array", "description": "the whole file is a single JSON array, closed when the file rotates", "type": "string" } ] }, "FileOutputConfig": { "description": "Writes each batch to files in a directory on the server.\n\nThe directory comes from a `file` connection and the `path` below is\nrelative to it; the server's `--data-dir` is what both are confined to, so a\nserver started without that flag cannot write files at all. Names are\ngenerated rather than configured — `-.`, which\nsorts chronologically and cannot collide across rotations.\n\nMeant for local development and testing. The object-store output is what\nthis shape is being built towards for anything else.", "properties": { "connection": { "description": "name of the file connection to write under — see \"connections\" in the\nreadme. The root directory lives there; the path below is this output's\nown.", "type": "string", "x-connection": "file" }, "format": { "anyOf": [ { "$ref": "#/components/schemas/FileFormat" }, { "type": "null" } ], "description": "how the messages are laid out. Defaults to `ndjson`." }, "path": { "description": "directory to write into, relative to the connection's root, e.g.\n`orders`. Must stay inside the root: an absolute path or one containing\n`..` is refused rather than trimmed.", "type": "string" }, "rotate": { "anyOf": [ { "$ref": "#/components/schemas/RotationConfig" }, { "type": "null" } ], "description": "when to close a file and start the next one. Without this, one file per\nrun." } }, "required": [ "connection", "path" ], "title": "file", "type": "object" }, "FilterTransformConfig": { "description": "Drops messages that don't match a condition, and drops the whole batch if\nnone of them do. Pick either the `Numeric` or the `String` form — the fields\ndiffer because the comparisons do.", "oneOf": [ { "properties": { "Numeric": { "properties": { "field": { "description": "the field to filter on", "type": "string", "x-message-field": true }, "operator": { "$ref": "#/components/schemas/NumericFilterOperatorKind" }, "value": { "format": "double", "type": "number" } }, "required": [ "field", "operator", "value" ], "type": "object" } }, "required": [ "Numeric" ], "type": "object" }, { "properties": { "String": { "properties": { "field": { "type": "string", "x-message-field": true }, "operator": { "$ref": "#/components/schemas/StringFilterOperatorKind" }, "value": { "type": "string" } }, "required": [ "field", "operator", "value" ], "type": "object" } }, "required": [ "String" ], "type": "object" } ], "title": "filter", "type": "object" }, "HistoryBucket": { "description": "One time unit's worth of counting, as the store keeps it and the API serves\nit.\n\nDeliberately the same three questions the card chart asks\n(`frontend::stats::Bucket`) plus the one it can't answer from a sampled\nfeed: how many failures there were. Counts, not messages — which is what\nmakes a bucket 32 bytes whatever the pipeline is carrying, and what makes\nkeeping a day of them cost less than one message of most real payloads.", "properties": { "errors": { "description": "Failures at any stage during the bucket. The *true* count, not the\nthrottled one: this is a counter, so suppressing a repeat in the log\ndoesn't hide it here.", "format": "uint64", "minimum": 0, "type": "integer" }, "inbound": { "description": "Messages that arrived at the pipeline's inputs during the bucket.", "format": "uint64", "minimum": 0, "type": "integer" }, "outbound": { "description": "Messages that came out of the transform chain and were handed to the\noutputs during the bucket.\n\nCounted once per batch, not once per output — it is what the pipeline\n*produced*, which is the same thing the card chart's outbound bar counts\noff the `Stage::Output` events, so the two agree. A transform that\nchanges cardinality is what makes this differ from `inbound`; a failing\noutput is not, and shows up in `errors` instead.", "format": "uint64", "minimum": 0, "type": "integer" }, "start": { "description": "Where the bucket starts, in seconds since the epoch. Always a multiple\nof the resolution's width.", "format": "uint64", "minimum": 0, "type": "integer" } }, "required": [ "start", "inbound", "outbound", "errors" ], "type": "object" }, "HttpAuthConfig": { "description": "A credential carried in a header — checked by the `http` input on a post to\na pipeline's endpoint, and presented by the `http` output on a request it\nsends.\n\nOne type for both directions because it is one fact: a fixed string in a\nnamed header. The two halves read it differently — the input compares what\narrived against this, the output sets it — and only the input has the rule\nabout `ALLOWED_HEADERS`, since only the input can write a header into the\nmessages.\n\nThis is the **data plane's** own credential and has nothing to do with the\naccounts in the settings file: those are people signing in to look at and\nedit the graph, this is one system pushing data into one pipeline. A machine\nposting readings should not need an account that can rewrite the config, and\na person with such an account should not thereby be able to post readings.\n\nThe token is a fixed string the sender repeats on every request, which makes\nit **only as private as the transport**. kayak speaks plain HTTP; putting\nTLS in front of it is the deployment's job, and without that the token is\nreadable by anything on the path. It is the same trade every log-ingest API\nmakes, and worth making deliberately rather than by accident.", "oneOf": [ { "description": "A token in the standard `Authorization` header, as\n`Authorization: Bearer `. The one to reach for unless the system\non the other end can't use that header.", "properties": { "token": { "$ref": "#/components/schemas/Secret", "description": "the token. A `${NAME}` reference, so the config file holds the name\nand the secret store holds the value." }, "type": { "const": "bearer", "type": "string" } }, "required": [ "type", "token" ], "type": "object" }, { "description": "A fixed value in a header of your choosing — for webhook senders and\nreceivers that can't use `Authorization` but can carry a header of their\nown, which is most of them.", "properties": { "name": { "description": "the header's name, matched case-insensitively on the way in. On an\n`http` input it may not be one of the headers an `envelope` passes\nthrough, since that would write the credential into the messages.", "type": "string" }, "type": { "const": "header", "type": "string" }, "value": { "$ref": "#/components/schemas/Secret", "description": "the exact value that header must have. A `${NAME}` reference, as\nabove." } }, "required": [ "type", "name", "value" ], "type": "object" } ] }, "HttpBodyKind": { "description": "What the body of one request from an `http` output holds.\n\nA closed set of two, and the choice is the receiving API's rather than a\ntuning knob: an ingest endpoint that takes an array wants `batch`, a webhook\nthat takes one event per call wants `message`. There is no third spelling\n(an envelope with a count, say) because that is the receiver's shape, and\nshaping the request is the http transform's outstanding work, not this\ncomponent's.", "oneOf": [ { "const": "batch", "description": "The whole batch as one JSON array, in one request. One round trip per\nbatch however many messages it holds, which is why it is the default.", "type": "string" }, { "const": "message", "description": "One request per message, each body the message itself. Requests go out\nin order and the first failure fails the batch, so the messages after it\nare not sent — the same all-or-nothing a broker publish loop has.", "type": "string" } ] }, "HttpInputConfig": { "description": "Accepts messages posted to this pipeline's own endpoint,\n`POST /api/pipelines/{id}/messages` — the pipeline is the receiving end of\nan http API rather than something that reaches out to a broker.\n\nThe endpoint is derived from the pipeline's id and appears as soon as the\npipeline is running; nothing is configured about it here. The body is one\nJSON message or an array of them, and an array arrives as one batch. A\npipeline can only have one of these — two would share an endpoint, and which\nof them a request went to would be a coin toss — so a second one fails to\nbuild.", "properties": { "auth": { "anyOf": [ { "$ref": "#/components/schemas/HttpAuthConfig" }, { "type": "null" } ], "description": "what a post must present to be accepted. Absent — the default — means\nthe endpoint takes anything that reaches it, which is what every\npipeline with an `http` input has always done." }, "capacity": { "description": "how many posted batches may queue up ahead of the pipeline before it\nstarts refusing them with a `503`. Defaults to 1024. The queue is what\nlets a burst through; refusing past it is deliberate, since the\nalternative is holding a request open until the pipeline catches up.", "format": "uint", "minimum": 0, "type": [ "integer", "null" ] } }, "title": "http", "type": "object" }, "HttpOutputConfig": { "description": "Sends the batch to an http endpoint — the pipeline pushes its results at a\nwebhook or an ingest API rather than at a broker.\n\nThe counterpart of the `http` *input*, and the sending half of what the\n`http` transform does: the transform replaces the batch with the reply, this\none is the end of the chain and the reply's body is discarded. What is not\ndiscarded is its **status** — anything but a 2xx fails the batch, which is\nwhat makes a webhook that is rejecting the data show up on the card rather\nthan being written off as delivered.", "properties": { "auth": { "anyOf": [ { "$ref": "#/components/schemas/HttpAuthConfig" }, { "type": "null" } ], "description": "what this output presents to be allowed to send. Absent — the default —\nsends no credential at all, which is what an open webhook wants." }, "body": { "anyOf": [ { "$ref": "#/components/schemas/HttpBodyKind" }, { "type": "null" } ], "description": "what one request carries. Defaults to `batch`, which is one request per\nbatch." }, "timeout_seconds": { "description": "how long one request may take before it is given up on, in seconds.\nDefaults to 30. A batch whose request times out is a failed batch, so\nthis is also the longest a slow endpoint can hold the pipeline up.", "format": "uint64", "minimum": 0, "type": [ "integer", "null" ] }, "url": { "description": "endpoint to send to, e.g. `https://example.com/hooks/readings`", "type": "string" }, "verb": { "anyOf": [ { "$ref": "#/components/schemas/HttpVerb" }, { "type": "null" } ], "description": "http method. Defaults to `POST`. `GET` and `DELETE` are refused at build\ntime — an output exists to send the messages somewhere, and a method\nwith no body has nowhere to put them." } }, "required": [ "url" ], "title": "http", "type": "object" }, "HttpTransformConfig": { "description": "Posts the batch to an http endpoint as a JSON array and replaces it with the\nJSON array in the response — so the service on the other end is the\ntransform.", "properties": { "url": { "description": "endpoint to send the batch to", "type": "string" }, "verb": { "$ref": "#/components/schemas/HttpVerb", "description": "http method. Accepted but not honoured yet: every request is a POST." } }, "required": [ "url", "verb" ], "title": "http", "type": "object" }, "HttpVerb": { "description": "The http method an http transform sends with.\n\nA closed set rather than a `String` because it is one: a request is made\nwith one of these or it is not made at all, and typing the name of a method\ninto a box is a way of finding that out one round trip later than necessary.", "enum": [ "GET", "POST", "PUT", "PATCH", "DELETE" ], "type": "string" }, "IngestRequest": { "anyOf": [ { "description": "Several messages, delivered as one batch.", "items": true, "type": "array" }, { "description": "A single message, delivered as a batch of one." } ], "description": "What `POST /api/pipelines/{id}/messages` takes: one message, or an array of\nthem.\n\nUntagged, and the array arm comes first on purpose — a JSON array would\notherwise deserialize as [`IngestRequest::One`] holding an array, and posting\nten messages would put one message into the pipeline. There is no envelope\naround the messages because there is nothing to put in one: the pipeline is\nnamed by the path, and kayak has no schema to declare.", "title": "IngestRequest" }, "IngestResponse": { "description": "What came back from a post: how many messages were handed to the pipeline.\n\nIt says *accepted*, not *processed* — the batch is queued for the run loop\nand the response doesn't wait for it, so a 202 means the pipeline has the\nmessages, not that the outputs have written them.", "properties": { "accepted": { "format": "uint", "minimum": 0, "type": "integer" } }, "required": [ "accepted" ], "title": "IngestResponse", "type": "object" }, "InputConfig": { "oneOf": [ { "$ref": "#/components/schemas/DummyConfig", "properties": { "type": { "const": "dummy", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/HttpInputConfig", "properties": { "type": { "const": "http", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/KafkaConfig", "properties": { "type": { "const": "kafka", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/NatsConfig", "properties": { "type": { "const": "nats", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/PipelineConfig", "properties": { "type": { "const": "pipeline", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/MqttConfig", "properties": { "type": { "const": "mqtt", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/RedisConfig", "properties": { "type": { "const": "redis", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/OpcuaConfig", "properties": { "type": { "const": "opcua", "type": "string" } }, "required": [ "type" ], "type": "object" } ], "properties": { "ack": { "anyOf": [ { "$ref": "#/components/schemas/AckMode" }, { "type": "null" } ], "description": "when this input tells its broker a message is done with. Available on\nevery input kind in the schema, but only honoured by ones with a\nbroker-side notion of \"received\" vs \"delivered\" of their own (`kafka`,\nfor now) — an input with nothing to acknowledge refuses to build rather\nthan silently treating this as `on_receipt`. Defaults to `on_receipt`,\nwhich is what every input has always done. See \"acknowledgement modes\"\nin the guide." }, "buffer": { "anyOf": [ { "$ref": "#/components/schemas/BufferConfig" }, { "type": "null" } ], "description": "batch messages from this input before the transforms see them — by\ncount (`static`), by time (`tumbling`) or by whichever comes first\n(`batch`). Never emits an empty batch. Available on every input kind.\nNot to be confused with the `buffer` transform." }, "envelope": { "anyOf": [ { "$ref": "#/components/schemas/EnvelopeConfig" }, { "type": "null" } ], "description": "attach metadata about where each message came from — the subject, topic,\npartition and so on listed under \"metadata\" below. Available on every\ninput kind. Omit it and messages are passed on exactly as they arrive." } }, "type": "object" }, "KafkaConfig": { "description": "Consumes JSON messages from a kafka topic, each emitted as a batch of one.\n\nA payload that isn't JSON is skipped with a warning rather than taking the\npipeline down, same as the nats input. The consumer connects on the first\nread and joins a consumer group, so kafka remembers where this pipeline got\nto between restarts.", "properties": { "connection": { "description": "name of the kafka connection to consume from — see \"connections\" in the\nreadme. The brokers are declared once, in the connections file, rather\nthan repeated in every pipeline reading from the same cluster.", "type": "string", "x-connection": "kafka" }, "group": { "description": "consumer group id. Kafka tracks the read position per group, so two\npipelines sharing a group split the topic between them, and two with\ndifferent groups each get every message.", "type": "string" }, "max_batch": { "description": "most messages to put in one batch. Defaults to 1 — one message per\nbatch, which is what this input has always done.\n\nRaising it only ever coalesces records that had *already arrived*: the\ninput still returns as soon as it has one, so an idle topic is no slower\nthan it was. It is worth raising when a consumer is catching up on a\nbacklog, where one-message batches make the run loop, the transforms and\nevery downstream pipeline do their per-batch work a hundred times over.", "format": "uint", "minimum": 0, "type": [ "integer", "null" ] }, "start_at": { "anyOf": [ { "$ref": "#/components/schemas/KafkaStartAt" }, { "type": "null" } ], "description": "where to start when the group has no committed position yet: `earliest`\nreplays the topic from the beginning, `latest` only sees new messages.\nDefaults to `latest`." }, "topic": { "description": "the topic to consume from", "type": "string" } }, "required": [ "connection", "topic", "group" ], "title": "kafka", "type": "object" }, "KafkaConnection": { "description": "A kafka cluster: the brokers, and eventually whatever it takes to\nauthenticate against them.", "properties": { "brokers": { "$ref": "#/components/schemas/Secret", "description": "comma-separated broker list, e.g. `localhost:9092`. May reference\nsecrets as `${NAME}` — see \"secrets\" in the readme." } }, "required": [ "brokers" ], "title": "kafka", "type": "object" }, "KafkaOutputConfig": { "description": "Publishes every message in the batch to a kafka topic, one message per\nrecord. Records are sent without a key, so they round-robin across the\ntopic's partitions.", "properties": { "connection": { "description": "name of the kafka connection to publish to — see \"connections\" in the\nreadme.", "type": "string", "x-connection": "kafka" }, "topic": { "description": "the topic to publish to", "type": "string" } }, "required": [ "connection", "topic" ], "title": "kafka", "type": "object" }, "KafkaStartAt": { "description": "Where a new consumer group starts reading.", "enum": [ "earliest", "latest" ], "type": "string" }, "KeepPolicy": { "description": "Whether a `map` passes through the fields it wasn't told about.", "oneOf": [ { "const": "all", "description": "The message is passed through and the mappings are laid over it. The\ndefault, because it is the one that doesn't quietly discard data: a map\nthat renamed one field would otherwise throw the rest of the message\naway.", "type": "string" }, { "const": "mapped", "description": "Only the fields the mappings wrote come out — a projection. This is what\nprepares a message for an output with a shape of its own (a `postgres`\ntable, an `s3` part), and it is also what sweeps up the intermediate\nfields a chained arithmetic leaves behind.", "type": "string" } ] }, "LayoutFile": { "description": "The whole file: every pipeline someone has placed by hand, and every edge whose\nroute they have adjusted.\n\nAbsent ids are the normal case, not a gap to be filled — the canvas lays\nthose out itself. A pipeline is added here the first time it is dragged and\nremoved when the arrangement is reset.", "properties": { "edges": { "description": "Kept sorted by `(from, to)` — see [`LayoutFile::set_edge_offset`] — so\nthe file stays diffable.", "items": { "$ref": "#/components/schemas/EdgeLayout" }, "type": "array" }, "pipelines": { "additionalProperties": { "$ref": "#/components/schemas/PipelineLayout" }, "default": {}, "type": "object" }, "version": { "default": 1, "format": "uint32", "minimum": 0, "type": "integer" } }, "title": "LayoutFile", "type": "object" }, "Literal": { "description": "A literal value written by a `constant`, or standing in for a field that\nisn't there.\n\nSpelled as a tagged union rather than as a bare JSON value because an\nuntyped `Value` field reflects as a box to hand-write JSON into, and one of\nthose in a form is a field the user has to already know the answer for.\nTagging it means the form asks which kind of value and then offers the right\ncontrol.", "oneOf": [ { "description": "A string.", "properties": { "type": { "const": "text", "type": "string" }, "value": { "description": "the text", "type": "string" } }, "required": [ "type", "value" ], "type": "object" }, { "description": "A number.", "properties": { "type": { "const": "number", "type": "string" }, "value": { "description": "the number", "format": "double", "type": "number" } }, "required": [ "type", "value" ], "type": "object" }, { "description": "True or false.", "properties": { "type": { "const": "boolean", "type": "string" }, "value": { "description": "the flag", "type": "boolean" } }, "required": [ "type", "value" ], "type": "object" }, { "description": "JSON null — an explicit \"nothing\", as against leaving the field out.", "properties": { "type": { "const": "null", "type": "string" } }, "required": [ "type" ], "type": "object" } ] }, "LoginRequest": { "description": "What `POST /api/auth/login` takes.\n\nThe password is a plain `String` and not a\n[`Secret`](crate::config::Secret), which is the opposite of every other\npassword field in kayak and deliberately so: a `Secret` holds a `${NAME}`\n*reference* to a credential, and this is the credential itself, typed into a\nlogin box a moment ago. It exists for the length of one request and is never\nstored, serialized back or logged.", "properties": { "password": { "type": "string" }, "username": { "type": "string" } }, "required": [ "username", "password" ], "title": "LoginRequest", "type": "object" }, "MapMissingPolicy": { "description": "What `map` does about a message that doesn't carry a field a mapping reads.\n\nIt has its own set rather than sharing the reducer's `MissingFieldPolicy` or\n`recall`'s `RecallMissingPolicy` for one specific reason: `skip` already\nmeans two different things in those two (\"leave this message out of this\naggregation\" and \"drop the message\"), and a third reading of the same word\nwould make the config file unreadable. So the arm that leaves the target\nfield unwritten is called `omit`, and there is deliberately no arm that\ndrops the message — that is what `filter` is for.", "oneOf": [ { "const": "error", "description": "Fail the pipeline. The default, on the reducer's argument: a mapping\nthat silently produced nothing is wrong in a way nothing downstream can\nsee. Say `omit`, or give that one mapping a `default`, to mean it.", "type": "string" }, { "const": "omit", "description": "Leave the target field unwritten, as though the mapping weren't there.", "type": "string" }, { "const": "null", "description": "Write the target field as `null`.", "type": "string" } ] }, "MapTransformConfig": { "description": "Rewrites the shape of every message: renames, promotions, constants, casts\nand projections, applied in order.\n\nEach entry in `mappings` reads fields from the message and writes one field\nback, and **later entries see what earlier ones wrote** — so an intermediate\nvalue is just a mapping whose target a later mapping reads (and, under\n`keep: all`, a `drop` takes away again).\n\nReads are dotted paths, like everywhere else. Writes are too: an `as` of\n`sensor.id` puts the value inside a `sensor` object, creating it if it isn't\nthere.\n\nThe message is passed through unchanged, with the mappings laid over it,\nunless `keep` says otherwise. One message always comes out — this never\ndrops one, and never makes two. Reach for `filter` or `splitter` for those.", "properties": { "keep": { "$ref": "#/components/schemas/KeepPolicy", "description": "whether fields nothing mapped survive" }, "mappings": { "description": "what to write, in the order it is written. At least one, and no two may\nwrite the same field.", "items": { "$ref": "#/components/schemas/Mapping" }, "type": "array" }, "on_missing": { "$ref": "#/components/schemas/MapMissingPolicy", "description": "what to do about a message missing a field a mapping reads. A `default`\non the mapping itself is answered first, and is the better way to say\nthat one particular field is expected to be absent." } }, "required": [ "mappings" ], "title": "map", "type": "object" }, "Mapping": { "description": "One field written onto the message, and where its value comes from.\n\nA tagged union rather than one struct with a great many optional fields, for\nthe reason `Condition` gives: a list of these has to render as a form, and a\npile of boxes of which four are relevant offers no way to say which four.\nHere the tag is picked first and the rest of the row follows from it.", "oneOf": [ { "description": "Takes a value from one field and writes it to another — a rename, or a\npromotion of something out of a nested object (`_meta.subject` →\n`subject`).", "properties": { "as": { "description": "the field to write. Left out, it is `from`'s last segment, which is\nthe reading that makes promoting a nested value the short spelling.", "type": [ "string", "null" ] }, "default": { "anyOf": [ { "$ref": "#/components/schemas/Literal" }, { "type": "null" } ], "description": "what to write when `from` isn't there, instead of applying\n`on_missing`" }, "from": { "description": "the field to read — a dotted path, like anywhere else", "type": "string" }, "type": { "const": "copy", "type": "string" } }, "required": [ "type", "from" ], "type": "object" }, { "description": "Writes a fixed value — the environment, the site, the name of the feed.", "properties": { "as": { "description": "the field to write it to", "type": "string" }, "type": { "const": "constant", "type": "string" }, "value": { "$ref": "#/components/schemas/Literal", "description": "the value to write" } }, "required": [ "type", "value", "as" ], "type": "object" }, { "description": "Writes the first of several fields that the message actually carries.\n\nThis is what merging two sources that spell one thing differently comes\nto, and it needs no expression language to say.", "properties": { "as": { "description": "the field to write the first value found to", "type": "string" }, "default": { "anyOf": [ { "$ref": "#/components/schemas/Literal" }, { "type": "null" } ], "description": "what to write when none of them is there" }, "from": { "description": "the fields to try, in order. At least two — with one, this is a\n`copy`.", "items": { "type": "string" }, "type": "array" }, "type": { "const": "coalesce", "type": "string" } }, "required": [ "type", "from", "as" ], "type": "object" }, { "description": "Converts a value from one JSON shape to another — the string `\"12.5\"` to\nthe number `12.5`, an epoch second to a timestamp, a string of embedded\nJSON to the thing it describes.\n\nThis is the one place in kayak where coercion is legal, and that is the\ndivision of labour: a `postgres` column mapping *checks* a value and\nnever converts it, so a stream that needs converting says so once, here,\nrather than at each of three outputs.", "properties": { "as": { "description": "the field to write. Left out, it is `from`'s last segment — so\ncasting a field in place is `{\"from\": \"value\", \"to\": \"float\"}`.", "type": [ "string", "null" ] }, "default": { "anyOf": [ { "$ref": "#/components/schemas/Literal" }, { "type": "null" } ], "description": "what to write when `from` isn't there. A value that *is* there and\nwon't convert is an error either way — that is a stream that isn't\nwhat the config says it is, not a missing field." }, "from": { "description": "the field to read", "type": "string" }, "to": { "$ref": "#/components/schemas/CastType", "description": "what to convert it to" }, "type": { "const": "cast", "type": "string" } }, "required": [ "type", "from", "to" ], "type": "object" }, { "description": "Joins fields and literal text into one string.\n\nMostly earns its place because `group_by` takes a list of fields and has\nno composite key: building `site/machine` as a field is the only way to\ngroup on the pair.", "properties": { "as": { "description": "the field to write the joined string to", "type": "string" }, "parts": { "description": "the pieces, in order. At least one.", "items": { "$ref": "#/components/schemas/ConcatPart" }, "type": "array" }, "type": { "const": "concat", "type": "string" } }, "required": [ "type", "parts", "as" ], "type": "object" }, { "description": "One arithmetic operation on two numbers, each of them a field or a\nliteral.\n\nOne operation, deliberately: `(f - 32) / 1.8` is two of these through an\nintermediate field, and the fact that three or four steps read badly is\ninformation rather than a defect — it is where this stops being\nconfiguration.", "properties": { "as": { "description": "the field to write the answer to", "type": "string" }, "left": { "$ref": "#/components/schemas/Operand", "description": "the left-hand operand" }, "operator": { "$ref": "#/components/schemas/ArithmeticOperator", "description": "what to do with them" }, "right": { "$ref": "#/components/schemas/Operand", "description": "the right-hand operand" }, "type": { "const": "arithmetic", "type": "string" } }, "required": [ "type", "left", "operator", "right", "as" ], "type": "object" }, { "description": "Takes fields off the message.\n\nThe counterpart of the in-band envelope: metadata that a `group_by`\nneeded is rarely metadata an output wants, and this is what takes it\nback off before the message leaves. Removing a field that isn't there is\nnot an error — `on_missing` doesn't apply.", "properties": { "from": { "description": "the fields to remove. At least one.", "items": { "type": "string" }, "type": "array" }, "type": { "const": "drop", "type": "string" } }, "required": [ "type", "from" ], "type": "object" } ] }, "MetaFieldDoc": { "description": "One metadata field an input attaches, and what it holds.", "properties": { "description": { "type": "string" }, "name": { "description": "The field's name inside the metadata object.", "type": "string" } }, "required": [ "name", "description" ], "type": "object" }, "MissingColumnPolicy": { "description": "What to do about a message that doesn't carry a column's field.\n\nA field that is present but `null` counts as missing — the same reading the\nreducer's [`crate::config::MissingFieldPolicy`] takes, and the same fact\nsaid two ways.", "oneOf": [ { "const": "null", "description": "Write `NULL`. The default for a nullable column: a stream where some\nmessages carry a field and some don't is the ordinary case, and a table\nthat says so is a table you can still query.", "type": "string" }, { "const": "error", "description": "Fail the pipeline. The default for a column declared `\"nullable\": false`,\nsince there is nothing else such a column could do.", "type": "string" }, { "const": "skip_row", "description": "Leave the whole message out of the table. Nothing about that row is\nwritten, including the columns that *were* present.", "type": "string" } ] }, "MissingFieldPolicy": { "description": "What to do about a message that doesn't carry a field being aggregated or\ngrouped by. A field present but `null` counts as missing — it is the same\nfact said two ways.", "oneOf": [ { "const": "error", "description": "Fail the pipeline. The default, because a sum over \"whichever messages\nhappened to have the field\" is wrong in a way nothing downstream can see.", "type": "string" }, { "const": "skip", "description": "Leave that message out of that one aggregation. An aggregation left with\nno values at all reports `null` (or `0`, for the counts).", "type": "string" } ] }, "MqttConfig": { "description": "Subscribes to an mqtt topic — or a topic *filter*, since mqtt's `+` and `#`\nwildcards are valid here. Each message is parsed as JSON and emitted as a\nbatch of one; a payload that isn't JSON is skipped with a warning rather\nthan taking the pipeline down, the same rule every other input follows.\n\nThe connection is opened on the first read, and a stable client id is\nderived from the pipeline's id and this topic — not configurable, since\nnothing about it is a choice this pipeline needs to make and getting it\nwrong (two inputs sharing one id) silently drops one of them.", "properties": { "connection": { "description": "name of the mqtt connection to subscribe on — see \"connections\" in the\nreadme. The broker it points at is declared once, in the connections\nfile, rather than repeated in every pipeline that uses it.", "type": "string", "x-connection": "mqtt" }, "max_batch": { "description": "most messages to put in one batch. Defaults to 1 — one message per\nbatch, which is what this input has always done.\n\nRaising it only ever coalesces messages that had *already arrived*: the\ninput still returns as soon as it has one, so a quiet topic is no\nslower than it was.", "format": "uint", "minimum": 0, "type": [ "integer", "null" ] }, "qos": { "anyOf": [ { "$ref": "#/components/schemas/MqttQos" }, { "type": "null" } ], "description": "the quality of service to subscribe with. Defaults to `at_most_once`.\n`ack: on_delivery` needs at least `at_least_once` here — a QoS-0\nsubscription has nothing for it to acknowledge." }, "topic": { "description": "the topic, or topic filter, to subscribe to", "type": "string" } }, "required": [ "connection", "topic" ], "title": "mqtt", "type": "object" }, "MqttConnection": { "description": "An mqtt broker.\n\nPlaintext TCP only for now — there is no TLS field here yet, and that is a\ndeliberate gap (see `docs/roadmap.md`) rather than an oversight: a CA\ncertificate needs somewhere to live (a `Secret`? a file path resolved\nagainst `--data-dir`?) and that question deserves its own pass rather than\na field bolted on to get this connection working.", "properties": { "host": { "description": "broker hostname, e.g. `localhost`", "type": "string" }, "password": { "anyOf": [ { "$ref": "#/components/schemas/Secret" }, { "type": "null" } ], "description": "that username's password. May reference secrets as `${NAME}` — see\n\"secrets\" in the readme, and prefer a reference to a literal here." }, "port": { "description": "broker port. Defaults to 1883, mqtt's conventional plaintext port.", "format": "uint16", "maximum": 65535, "minimum": 0, "type": [ "integer", "null" ] }, "username": { "anyOf": [ { "$ref": "#/components/schemas/Secret" }, { "type": "null" } ], "description": "username to connect with, if the broker requires one. Must be set\ntogether with `password` or not at all." } }, "required": [ "host" ], "title": "mqtt", "type": "object" }, "MqttOutputConfig": { "description": "Publishes every message in the batch to an mqtt topic, one message per\npublish.\n\nA stable client id is derived from the pipeline's id and this topic, the\nsame as the mqtt input — not configurable, for the same reason.", "properties": { "connection": { "description": "name of the mqtt connection to publish on — see \"connections\" in the\nreadme.", "type": "string", "x-connection": "mqtt" }, "qos": { "anyOf": [ { "$ref": "#/components/schemas/MqttQos" }, { "type": "null" } ], "description": "the quality of service to publish with. Defaults to `at_most_once`." }, "retain": { "description": "ask the broker to keep this as the topic's *retained* message, handed\nto every future subscriber immediately on subscribe. Defaults to false.", "type": [ "boolean", "null" ] }, "topic": { "description": "the topic to publish to", "type": "string" } }, "required": [ "connection", "topic" ], "title": "mqtt", "type": "object" }, "MqttQos": { "description": "The delivery guarantee to ask for on an mqtt subscribe or publish, spelled\nthe way mqtt itself names them rather than as the bare numbers `0`/`1`/`2`.", "oneOf": [ { "const": "at_most_once", "description": "fire and forget — the broker never resends and there is no ack of any\nkind. The default.", "type": "string" }, { "const": "at_least_once", "description": "the broker resends until acknowledged, so a message may arrive more\nthan once. Required for an input's `ack: on_delivery` to mean anything\n— see \"acknowledgement modes\" in the guide.", "type": "string" }, { "const": "exactly_once", "description": "the broker's four-part handshake that guarantees exactly one delivery.\nThe most expensive of the three; reach for `at_least_once` unless a\nduplicate would actually be wrong.", "type": "string" } ] }, "NatsConfig": { "description": "Subscribes to a nats subject. Each message is parsed as JSON and emitted as\na batch of one; a payload that isn't JSON is skipped with a warning rather\nthan taking the pipeline down. The connection is opened on the first read.", "properties": { "connection": { "description": "name of the nats connection to subscribe on — see \"connections\" in the\nreadme. The server it points at is declared once, in the connections\nfile, rather than repeated in every pipeline that uses it.", "type": "string", "x-connection": "nats" }, "max_batch": { "description": "most messages to put in one batch. Defaults to 1 — one message per\nbatch, which is what this input has always done.\n\nRaising it only ever coalesces messages that had *already arrived*: the\ninput still returns as soon as it has one, so a quiet subject is no\nslower than it was.", "format": "uint", "minimum": 0, "type": [ "integer", "null" ] }, "subject": { "description": "the subject to subscribe to", "type": "string" } }, "required": [ "connection", "subject" ], "title": "nats", "type": "object" }, "NatsConnection": { "description": "A nats server, or a cluster of them.", "properties": { "urls": { "$ref": "#/components/schemas/Secret", "description": "connection url, e.g. `nats://localhost:4222`. May reference secrets as\n`${NAME}` — see \"secrets\" in the readme." } }, "required": [ "urls" ], "title": "nats", "type": "object" }, "NatsOutputConfig": { "description": "Publishes every message in the batch to a nats subject, one message per\npublish.", "properties": { "connection": { "description": "name of the nats connection to publish on — see \"connections\" in the\nreadme.", "type": "string", "x-connection": "nats" }, "subject": { "description": "the subject to publish to", "type": "string" } }, "required": [ "connection", "subject" ], "title": "nats", "type": "object" }, "NumericFilterOperatorKind": { "description": "How a number is compared to the one in the config.", "enum": [ "greater_than", "less_than", "equal_to" ], "type": "string" }, "OpcuaBrowseConfig": { "description": "Everything under a node in the server's address space, found by browsing it\nwhen the pipeline starts.\n\nThe convenient half of naming nodes, and the one with a cost worth knowing:\nwhat this pipeline reads is then decided by the server's address space *at\nthe moment the pipeline starts*, so a tag added to the machine tomorrow is\npicked up by a restart and a tag removed silently stops arriving. An\nexplicit `nodes` list is the one that says in the config file exactly what\nis being read. The two combine — browse a folder and name the handful of\ntags elsewhere that belong with it.", "properties": { "depth": { "description": "how many levels below the root to follow. Defaults to 3, and there is\ndeliberately no spelling for \"all of them\": a browse of a plant server's\nwhole address space is thousands of nodes, and the pipeline that asked\nfor it would find that out by subscribing to them.", "format": "uint", "minimum": 0, "type": [ "integer", "null" ] }, "root": { "description": "id of the node to browse under, in the same notation as `node_id` —\ntypically a folder, e.g. `ns=2;s=Machine1`. Every *variable* found\nbeneath it is subscribed to; folders and objects are followed, not\nsubscribed.", "type": "string" } }, "required": [ "root" ], "title": "opcua browse", "type": "object" }, "OpcuaConfig": { "description": "Subscribes to variables on an OPC UA server, one message per value change.\n\nThe server pushes: this creates a subscription with a monitored item per\nnode and is told when a value changes, rather than reading them round-robin\non a timer. `publish_interval_ms` is how often the server may send, not how\noften it samples — a tag that doesn't move produces no messages at all.\n\nEach message is one reading, and carries the tag as well as the value:\n\n```json\n{\n \"node\": \"ns=2;s=Machine1.Temperature\",\n \"name\": \"temperature\",\n \"value\": 21.5,\n \"status\": \"Good\",\n \"source_timestamp\": \"2026-01-01T12:00:00.123Z\",\n \"server_timestamp\": \"2026-01-01T12:00:00.130Z\"\n}\n```\n\n`status` is the reading's own quality and is **always present** — a sensor\nthat has failed reports `Bad...` with a `null` value rather than going\nquiet, and a pipeline that acted on those as if they were readings would be\nacting on nothing. `source_timestamp` is when the *device* says the value\nwas produced, which is the one to reduce or partition by; the envelope's\n`received_at` is when kayak read it, and on a slow link those are not the\nsame instant.\n\nThe nodes are named by `nodes`, or found by `browse`, or both — one of them\nis required, since an input with nothing to monitor would sit silent\nforever. A node named twice is subscribed to once.", "properties": { "browse": { "anyOf": [ { "$ref": "#/components/schemas/OpcuaBrowseConfig" }, { "type": "null" } ], "description": "a node to browse, subscribing to every variable found under it." }, "connection": { "description": "name of the opcua connection to subscribe on — see \"connections\" in the\nreadme. The server it points at is declared once, in the connections\nfile, rather than repeated in every pipeline that uses it.", "type": "string", "x-connection": "opcua" }, "deadband": { "description": "how far a value must move before the server reports it, in the value's\nown units. Absent reports every change, however small — which on an\nanalogue signal is every sample, since the last digit is always moving.\n\nThis is applied by the *server*, so it saves the network and this\npipeline alike. It only applies to numeric nodes; a string or a boolean\nis reported on every change whatever this says.", "format": "double", "type": [ "number", "null" ] }, "max_batch": { "description": "most messages to put in one batch. Defaults to 1 — one message per\nbatch, which is what every other input does unless asked otherwise.\n\nWorth raising here more than elsewhere: one publish from the server\ncarries every node that changed in the interval, so a subscription to\ntwo hundred tags at 1 Hz is two hundred batches a second through the run\nloop unless they are allowed to travel together. Raising it only ever\ncoalesces changes that had *already arrived*.", "format": "uint", "minimum": 0, "type": [ "integer", "null" ] }, "nodes": { "description": "the nodes to subscribe to, named one by one.", "items": { "$ref": "#/components/schemas/OpcuaNodeConfig" }, "type": "array" }, "publish_interval_ms": { "description": "how often the server may send a batch of changes, in milliseconds.\nDefaults to 1000. This bounds how long a change waits, not how often\nanything is measured.", "format": "uint64", "minimum": 0, "type": [ "integer", "null" ] }, "queue_size": { "description": "how many samples the server may hold for a node between publishes.\nDefaults to 1, which means a value that changes twice in one interval is\nreported once — the latest. Raise it, together with\n`sampling_interval_ms`, when every sample matters rather than the\ncurrent value.", "format": "uint32", "minimum": 0, "type": [ "integer", "null" ] }, "sampling_interval_ms": { "description": "how often the server should *look* at each node, in milliseconds.\nAbsent asks the server to sample at the publishing interval, which is\nwhat it does by default; a smaller value here is what fills a queue with\nintermediate readings between two publishes.", "format": "uint64", "minimum": 0, "type": [ "integer", "null" ] } }, "required": [ "connection" ], "title": "opcua", "type": "object" }, "OpcuaConnection": { "description": "An OPC UA server, as one client session connects to it.\n\nThe endpoint is the whole of \"what the system is\" here — an OPC UA server\nexposes one address space at one url, and *which nodes* a pipeline reads out\nof it is the component's business, exactly as a topic is on a kafka\nconnection.\n\n**Plaintext and anonymous or username/password only.** There is no security\npolicy field and no certificate: an OPC UA session can be signed and\nencrypted, and that is worth having, but it needs a client certificate,\nsomewhere for it to live and a server trust list — the same question\n[`MqttConnection`]'s missing TLS raises, one size larger. It gets its own\npass (see `docs/roadmap.md`) rather than a field bolted on here, and until\nthen this refuses to pretend: the session is `SecurityPolicy::None`, so\ncredentials cross the wire in the clear and belong on a network you trust.\n\nOne consequence is visible in the log and is not a fault: the OPC UA client\nprints two errors about a missing *application instance certificate* when a\nsession is opened. kayak has none by design, and an unencrypted session\nneeds none — a pipeline that logs those and then reports readings is\nworking.", "properties": { "endpoint": { "$ref": "#/components/schemas/Secret", "description": "endpoint url, e.g. `opc.tcp://localhost:50000`. May reference secrets as\n`${NAME}` — see \"secrets\" in the readme.\n\nThis is connected to *directly*: kayak does not ask the server for its\nendpoint list first. Discovery is the usual way, and it is the usual way\nto fail — a server behind docker, NAT or a load balancer advertises the\nhostname it knows itself by, which is regularly not one the client can\nresolve. What is written here is what is dialled." }, "password": { "anyOf": [ { "$ref": "#/components/schemas/Secret" }, { "type": "null" } ], "description": "that username's password. May reference secrets as `${NAME}` — see\n\"secrets\" in the readme, and prefer a reference to a literal here." }, "username": { "anyOf": [ { "$ref": "#/components/schemas/Secret" }, { "type": "null" } ], "description": "username to sign in with, if the server requires one. Must be set\ntogether with `password` or not at all; without either, the session is\nanonymous." } }, "required": [ "endpoint" ], "title": "opcua", "type": "object" }, "OpcuaNodeConfig": { "description": "One node an `opcua` input subscribes to, and what the messages call it.", "properties": { "name": { "description": "what the messages from this node call it. Defaults to the node id\nitself, which is exact and unreadable; naming the tag here is what makes\nthe rest of the pipeline — a `group_by`, a column mapping — legible.", "type": [ "string", "null" ] }, "node_id": { "description": "the node's id, in OPC UA's own notation — `ns=2;s=Machine1.Temperature`\nfor a string identifier, `ns=2;i=1042` for a numeric one, `g=` for a\nguid and `b=` for an opaque one. A node id with no `ns=` is in\nnamespace 0, the server's own.", "type": "string" } }, "required": [ "node_id" ], "title": "opcua node", "type": "object" }, "OpenApiDocument": { "description": "An OpenAPI 3.1 document. See https://spec.openapis.org/oas/v3.1.0", "properties": { "components": { "type": "object" }, "info": { "type": "object" }, "openapi": { "type": "string" }, "paths": { "type": "object" } }, "required": [ "openapi", "paths" ], "title": "OpenApiDocument", "type": "object" }, "Operand": { "description": "One side of an [`Mapping::Arithmetic`]: a field to read, or a fixed number.", "oneOf": [ { "description": "A number read out of the message.", "properties": { "field": { "description": "the field to read — it has to hold a number", "type": "string" }, "type": { "const": "field", "type": "string" } }, "required": [ "type", "field" ], "type": "object" }, { "description": "A number written here in the config.", "properties": { "type": { "const": "value", "type": "string" }, "value": { "description": "the number", "format": "double", "type": "number" } }, "required": [ "type", "value" ], "type": "object" } ] }, "OutputConfig": { "oneOf": [ { "$ref": "#/components/schemas/StdoutOutputConfig", "properties": { "type": { "const": "stdout", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/FileOutputConfig", "properties": { "type": { "const": "file", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/S3OutputConfig", "properties": { "type": { "const": "s3", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/KafkaOutputConfig", "properties": { "type": { "const": "kafka", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/NatsOutputConfig", "properties": { "type": { "const": "nats", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/PostgresOutputConfig", "properties": { "type": { "const": "postgres", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/ClickhouseOutputConfig", "properties": { "type": { "const": "clickhouse", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/MqttOutputConfig", "properties": { "type": { "const": "mqtt", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/RedisOutputConfig", "properties": { "type": { "const": "redis", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/HttpOutputConfig", "properties": { "type": { "const": "http", "type": "string" } }, "required": [ "type" ], "type": "object" } ], "type": "object" }, "PipelineConfig": { "description": "Takes another pipeline's output as its input. This is what makes the\npipelines a graph: several pipelines can read from the same upstream, and it\nfans out to all of them. The upstream must already exist when this pipeline\nis created, so declare it earlier in the config file.", "properties": { "upstream": { "description": "id of the pipeline to read from", "type": "string", "x-pipeline-id": true } }, "required": [ "upstream" ], "title": "pipeline", "type": "object" }, "PipelineDryRunRequest": { "description": "Run a draft's transforms over some messages.", "properties": { "buckets": { "additionalProperties": { "additionalProperties": true, "type": "object" }, "description": "what that bucket should already hold, keyed as the pipeline keys it.\nThe warm-up a stateful chain would otherwise have to be talked through.", "type": "object" }, "messages": { "description": "the messages to put in, as one batch — which is what a `buffer` or a\n`reduce` will treat them as. A sample from `POST /api/inputs/sample` is\nwhat the UI puts here.", "items": true, "type": "array" }, "state": { "anyOf": [ { "$ref": "#/components/schemas/PipelineState" }, { "type": "null" } ], "description": "the pipeline's `state` block, if its transforms need one. The bucket it\nnames is created for this request alone." }, "transforms": { "default": [], "description": "the transforms, in order, exactly as they would be written in the\npipeline. An empty list is allowed and answers the trivial question.", "items": { "$ref": "#/components/schemas/TransformConfig" }, "type": "array" } }, "required": [ "messages" ], "title": "pipeline dry run request", "type": "object" }, "PipelineDryRunResponse": { "description": "What the chain did.", "oneOf": [ { "properties": { "buckets": { "additionalProperties": { "additionalProperties": true, "type": "object" }, "description": "what the private buckets ended up holding, so a `remember` is\nvisible rather than being a side effect nobody can see.", "type": "object" }, "outcome": { "const": "ran", "type": "string" }, "stages": { "description": "one entry per transform, in order.", "items": { "$ref": "#/components/schemas/StageResult" }, "type": "array" } }, "required": [ "outcome", "stages" ], "type": "object" }, { "properties": { "at": { "description": "which transform failed, by position.", "format": "uint", "minimum": 0, "type": "integer" }, "kind": { "type": "string" }, "message": { "type": "string" }, "outcome": { "const": "failed", "type": "string" }, "phase": { "$ref": "#/components/schemas/FailurePhase" }, "stages": { "description": "the stages that completed before it. Kept, rather than thrown away\nwith the failure: how far the messages got is half of what says\n*why* the failing transform failed.", "items": { "$ref": "#/components/schemas/StageResult" }, "type": "array" } }, "required": [ "outcome", "at", "kind", "phase", "message" ], "type": "object" } ], "title": "pipeline dry run response" }, "PipelineDto": { "description": "One pipeline as the API reports it: the id it is running under, the config\nit was built from, and whether its run loop is still alive.\n\nThe same wire shape the run loop's `PipelineView` serializes to — this is\nthe owned spelling of it, and the one the schema is generated from.", "properties": { "config": { "$ref": "#/components/schemas/Config" }, "id": { "type": "string" }, "status": { "$ref": "#/components/schemas/RunStatus", "default": "running", "description": "Where the run loop has got to. `#[serde(default)]` for the reason\n`UiEvent::ts` has one: a body from a server that predates the field\nreads as [`RunStatus::Running`], which is what every reader assumed\nbefore there was anything else it could be." } }, "required": [ "id", "config" ], "title": "PipelineDto", "type": "object" }, "PipelineHistory": { "description": "What `GET /api/pipelines/{id}/history` answers with.", "properties": { "bucket_secs": { "description": "How wide one bucket is, in seconds. Derivable from `resolution`, sent\nanyway so a chart can scale its axis without a table of constants.", "format": "uint64", "minimum": 0, "type": "integer" }, "buckets": { "description": "Oldest first, contiguous — gaps are filled with empty buckets rather\nthan omitted, so \"the pipeline stopped\" and \"the server wasn't asked\"\ndon't look alike. Empty when the pipeline has produced nothing yet.", "items": { "$ref": "#/components/schemas/HistoryBucket" }, "type": "array" }, "dropped_signatures": { "description": "Distinct failures dropped to stay under [`MAX_ERROR_SIGNATURES`], since\nthis pipeline started. Non-zero means the errors above are a selection,\nand is itself a diagnosis: a pipeline producing dozens of distinct\nfailure texts is usually one embedding a message id in each.", "format": "uint64", "minimum": 0, "type": "integer" }, "errors": { "description": "Distinct failures, most recently seen first, at most\n[`MAX_ERROR_SIGNATURES`] of them. Not scoped to the buckets' window:\na failure that started before the window is exactly the one worth\nshowing, and `first_seen` says so.", "items": { "$ref": "#/components/schemas/ErrorSignature" }, "type": "array" }, "resolution": { "$ref": "#/components/schemas/Resolution", "description": "Which ring this came from, echoed so a client that took the default\nknows what it got." } }, "required": [ "resolution", "bucket_secs", "buckets", "errors", "dropped_signatures" ], "title": "PipelineHistory", "type": "object" }, "PipelineLayout": { "description": "One card's place on the canvas, in surface coordinates (the same space the\nautomatic layout works in — pixels at zoom 1, origin at the top-left of the\ngraph).\n\n`height` is optional because the two ways a card gets its height are\ndifferent things: normally it is *measured* from the content it renders, and\nonly an explicit resize pins it. `None` means \"however tall it needs to be\",\nwhich is what you want a card to go back to when its config grows.", "properties": { "height": { "format": "double", "type": [ "number", "null" ] }, "width": { "format": "double", "type": "number" }, "x": { "format": "double", "type": "number" }, "y": { "format": "double", "type": "number" } }, "required": [ "x", "y", "width" ], "type": "object" }, "PipelineState": { "description": "A pipeline's binding to a bucket: which one, and what its messages are keyed\nby.\n\nThe key lives here rather than on the bucket because it is a property of\n*this stream* — the same machine id arrives as `_meta.machine_id` from a\nnats subscription and as `machine_id` after a reducer has flattened it, and\nboth are correct. The cost is that two pipelines sharing a bucket can key it\ndifferently with nothing to catch them, which is the sharp edge of sharing\nand is documented rather than prevented.", "properties": { "bucket": { "description": "name of the bucket this pipeline reads and writes — one of the ones\ndeclared under `state` at the top of the config. A pipeline naming a\nbucket that isn't declared fails to build.", "type": "string" }, "key": { "description": "the field whose value identifies the thing being remembered, e.g.\n`_meta.machine_id`. A dotted path like anywhere else.\n\nLeave it out for one bucket-wide value — which is the right answer for\nsomething there is only ever one of, and the wrong one for anything\nper-device.", "type": [ "string", "null" ] } }, "required": [ "bucket" ], "title": "pipeline state", "type": "object" }, "PortLayout": { "description": "Where an edge attaches to a card, once someone has said so by hand.\n\n`along` is measured from the start of the face — its left end for a top or\nbottom face, its top end for a left or right one — rather than as a fraction\nof it. A distance is what the drag meant: \"attach a card's width in from the\ncorner\" stays put when the card is made taller, where a fraction would slide.\n\n`side` is carried because the automatic router can change its mind: move a\ncard and an edge that left by the bottom may now leave by the side, and a\nposition measured along the old face means nothing on the new one. When they\ndisagree the stored position is ignored and the edge goes back to being\nplaced automatically, which is self-healing and needs no cleanup pass.", "properties": { "along": { "format": "double", "type": "number" }, "side": { "$ref": "#/components/schemas/Side" } }, "required": [ "side", "along" ], "type": "object" }, "PostgresConnection": { "description": "A postgres database, as one role connects to it.\n\nThe database and the role are part of the connection; the *table* is not —\nthat is what a particular output writes into, so it stays on the output.", "properties": { "database": { "description": "the database to connect to", "type": "string" }, "host": { "description": "server hostname, e.g. `localhost`", "type": "string" }, "password": { "$ref": "#/components/schemas/Secret", "description": "that role's password. May reference secrets as `${NAME}` — see\n\"secrets\" in the readme, and prefer a reference to a literal here." }, "port": { "description": "server port. Defaults to 5432.", "format": "uint16", "maximum": 65535, "minimum": 0, "type": [ "integer", "null" ] }, "user": { "description": "the role to connect as", "type": "string" } }, "required": [ "host", "database", "user", "password" ], "title": "postgres", "type": "object" }, "PostgresOutputConfig": { "description": "Inserts every message in the batch into a postgres table, one row per\nmessage.\n\nWith `columns`, each entry names a column, its type and the field to read —\n`{\"name\": \"temperature\", \"type\": \"float\", \"field\": \"reading.temp_c\"}`, and\n`field` defaults to the column's name. Without them the table gets a single\n`jsonb` column holding the whole message, which is what this output has\nalways done.\n\nThe table is created if it isn't there, from the columns above; set\n`create_table` to false for a table someone else owns. Creation never\n*alters* an existing table — a table whose shape has moved on fails the\ninsert with the server's own error rather than being migrated from a config\nfile.", "properties": { "columns": { "description": "which message field goes in which column. Leave it out to store each\nmessage whole, as JSON, in a `payload` column.", "items": { "$ref": "#/components/schemas/ColumnMapping" }, "type": "array" }, "connection": { "description": "name of the postgres connection to insert through — see \"connections\"\nin the readme. The host, database and role live there; the table below\nis this output's own.", "type": "string", "x-connection": "postgres" }, "create_table": { "description": "create the table on connect if it does not exist. Defaults to true.", "type": [ "boolean", "null" ] }, "indexes": { "description": "indexes to create with the table. Each names mapped columns, in order.", "items": { "$ref": "#/components/schemas/TableIndex" }, "type": "array" }, "on_extra_fields": { "$ref": "#/components/schemas/ExtraFieldPolicy", "description": "what to do about a message carrying fields no column reads" }, "primary_key": { "description": "the columns forming the created table's primary key. With none, the\ntable gets an `id` of its own and a `received_at` timestamp; naming one\nhere says the data carries its own identity and drops both.", "items": { "type": "string" }, "type": "array" }, "table": { "description": "the table to insert into, created if it does not exist. Optionally\nschema-qualified (`analytics.readings`); letters, digits and underscores\nonly, since it cannot be sent as a query parameter.", "type": "string" } }, "required": [ "connection", "table" ], "title": "postgres", "type": "object" }, "RecallMissingPolicy": { "description": "What `recall` does when the bucket has nothing for a message's key.\n\nIt has its own set rather than sharing the reducer's [`MissingFieldPolicy`]\nbecause the right default is the opposite one: every stateful pipeline has a\nwarm-up in which nothing has been remembered yet, so `error` would fail\nevery pipeline on startup, and it is `null` that has no counterpart there.", "oneOf": [ { "const": "skip", "description": "Drop the message. The default: a reading that can't be attributed to the\nthing it is about is usually noise, and passing it on unattributed makes\na reducer downstream lump every such message into one bogus group.", "type": "string" }, { "const": "null", "description": "Pass the message on with the missing names as `null`.", "type": "string" }, { "const": "error", "description": "Fail the pipeline. Only right when the bucket is filled by something\nthat has certainly run first.", "type": "string" } ] }, "RecallTransformConfig": { "description": "Writes values from the pipeline's state bucket onto every message, under the\nnames they were remembered by.\n\nThis is how a slow-moving fact — the unit being produced, the recipe in\nforce — reaches the fast stream that has to be attributed to it. The values\nland as top-level fields, so a `reducer` downstream can group by them\nwithout knowing where they came from.\n\nNeeds a `state` on the pipeline; it fails to build without one.", "properties": { "on_missing": { "$ref": "#/components/schemas/RecallMissingPolicy", "description": "what to do about a message whose key has nothing remembered under it yet" }, "recall": { "description": "the names to read out of the bucket, as `remember` wrote them. Each one\nis written onto the message under the same name.", "items": { "type": "string" }, "type": "array" } }, "required": [ "recall" ], "title": "recall", "type": "object" }, "RedisConfig": { "description": "Subscribes to a redis channel. Each message is parsed as JSON and emitted\nas a batch of one; a payload that isn't JSON is skipped with a warning\nrather than taking the pipeline down. The connection is opened on the\nfirst read.\n\nPlain `SUBSCRIBE`, not `PSUBSCRIBE` — a channel name is exact, the same\nchoice the nats input makes for a subject with no wildcard. Redis pub/sub\nhas no broker-side redelivery of any kind: an unsubscribed client simply\nmisses whatever was published while it was gone, and there is nothing an\nack could hold open — the same limitation `NatsConfig` has, for the same\nreason.", "properties": { "channel": { "description": "the channel to subscribe to", "type": "string" }, "connection": { "description": "name of the redis connection to subscribe on — see \"connections\" in\nthe readme. The server it points at is declared once, in the\nconnections file, rather than repeated in every pipeline that uses it.", "type": "string", "x-connection": "redis" }, "max_batch": { "description": "most messages to put in one batch. Defaults to 1 — one message per\nbatch, which is what this input has always done.\n\nRaising it only ever coalesces messages that had *already arrived*: the\ninput still returns as soon as it has one, so a quiet channel is no\nslower than it was.", "format": "uint", "minimum": 0, "type": [ "integer", "null" ] } }, "required": [ "connection", "channel" ], "title": "redis", "type": "object" }, "RedisConnection": { "description": "A redis server, or a cluster front-end that speaks the same protocol.\n\nUsed through its pub/sub commands (`SUBSCRIBE`/`PUBLISH`), the same shape\n[`NatsConnection`] is — one url, which may already carry a password —\nrather than the key-value store: there is no queue to consume from here,\nso a redis input has exactly the delivery guarantees a nats one does (see\n`RedisConfig`'s doc comment).", "properties": { "url": { "$ref": "#/components/schemas/Secret", "description": "connection url, e.g. `redis://localhost:6379` or\n`redis://:${REDIS_PASSWORD}@localhost:6379/0`. May reference secrets as\n`${NAME}` — see \"secrets\" in the readme." } }, "required": [ "url" ], "title": "redis", "type": "object" }, "RedisOutputConfig": { "description": "Publishes every message in the batch to a redis channel, one message per\npublish.", "properties": { "channel": { "description": "the channel to publish to", "type": "string" }, "connection": { "description": "name of the redis connection to publish on — see \"connections\" in the\nreadme.", "type": "string", "x-connection": "redis" } }, "required": [ "connection", "channel" ], "title": "redis", "type": "object" }, "ReduceFnKind": { "description": "How the values of one field are combined into a single answer.", "oneOf": [ { "const": "sum", "description": "The total. Numbers only.", "type": "string" }, { "const": "avg", "description": "The arithmetic mean. Numbers only.", "type": "string" }, { "const": "min", "description": "The smallest value. Numbers compare as numbers and strings\nalphabetically, which is what makes `min` over an ISO timestamp the\nearliest one.", "type": "string" }, { "const": "max", "description": "The largest value, comparing as `min` does.", "type": "string" }, { "const": "count", "description": "How many messages there were. The one function that needs no `field` —\ngiven one, it counts the messages that carry it instead.", "type": "string" }, { "const": "count_distinct", "description": "How many *different* values there were, compared by their JSON form.", "type": "string" }, { "const": "first", "description": "The value from the first message of the group, whatever type it is.", "type": "string" }, { "const": "last", "description": "The value from the last message of the group.", "type": "string" }, { "const": "collect", "description": "Every value, as an array, in the order they arrived.", "type": "string" }, { "const": "median", "description": "The middle value, or the mean of the middle two. Numbers only.", "type": "string" }, { "const": "stddev", "description": "The population standard deviation. Numbers only.", "type": "string" } ] }, "ReduceTransformConfig": { "description": "Reduces a batch to one message per group, carrying whatever was asked for\nabout it. Pair it with a buffer, or it will only ever see one message at a\ntime.\n\nWith no `group_by` the whole batch is one group and one message comes out;\nwith one, a message comes out per distinct combination of those fields, in\nthe order the groups were first seen. The emitted message carries the\ngrouping fields under their own names alongside the aggregations.\n\nEach aggregation is a `function`, the `field` to apply it to and the `as`\nname the answer is written under — `{\"function\": \"avg\", \"field\": \"value\",\n\"as\": \"mean\"}`. `count` is the one function that needs no `field`: without\none it counts the messages in the group, with one it counts the messages\nthat carried it.", "properties": { "aggregations": { "description": "what to compute. At least one, and each needs a distinct `as`.", "items": { "$ref": "#/components/schemas/Aggregation" }, "type": "array" }, "group_by": { "description": "the fields whose combination defines a group. Omit it to reduce the\nwhole batch at once.", "items": { "type": "string" }, "type": "array" }, "on_missing": { "$ref": "#/components/schemas/MissingFieldPolicy", "description": "what to do about a message missing one of the fields above" } }, "required": [ "aggregations" ], "title": "reducer", "type": "object" }, "RememberTransformConfig": { "description": "Writes values from matching messages into the pipeline's state bucket,\nkeyed by whatever the pipeline's `state.key` names.\n\nThe message itself is **passed on unchanged** — this is a tap on the stream,\nnot a filter. A transform called `remember` that quietly swallowed what it\nremembered would be a surprise, and the message is usually still wanted.\n\nNeeds a `state` on the pipeline; it fails to build without one.", "properties": { "remember": { "description": "what to take from a matching message. At least one, each with a distinct\n`as`.", "items": { "$ref": "#/components/schemas/Remembered" }, "type": "array" }, "when": { "description": "which messages to remember from — all of these have to match. Leave it\nout to remember from every message, which is right for a stream carrying\none kind of thing and wrong for one carrying several.", "items": { "$ref": "#/components/schemas/Condition" }, "type": "array" } }, "required": [ "remember" ], "title": "remember", "type": "object" }, "Remembered": { "description": "One thing to put in the pipeline's state bucket, and what to call it there.", "properties": { "as": { "description": "the name to remember it under, which is the name `recall` asks for it\nby. Two entries may not share one.", "type": "string" }, "field": { "description": "the field to take the value from", "type": "string" } }, "required": [ "field", "as" ], "type": "object" }, "Resolution": { "description": "Which ring a query is asking for. See the module docs for why there are two.", "oneOf": [ { "const": "fine", "description": "[`FINE_BUCKET_SECS`] a bucket, over [`FINE_WINDOW_SECS`]. What a card\nbackfills its live chart from.", "type": "string" }, { "const": "coarse", "description": "[`COARSE_BUCKET_SECS`] a bucket, over the configured retention. The\novernight record, and the default because that is what someone asking\nfor history at all is usually asking for.", "type": "string" } ] }, "Role": { "description": "What an account is allowed to do.\n\nTwo, and deliberately only two: the split that matters first is \"can change\nwhat the server is running\" against \"can watch it\". Anything finer — per\npipeline, per connection — needs a model of *which* resources, which is a\nmuch larger feature than a second role.", "oneOf": [ { "const": "admin", "description": "May do anything: create and delete pipelines and connections, save and\nrevert the config file, rearrange the canvas.", "type": "string" }, { "const": "read", "description": "May see everything and change nothing. The default for an account whose\n`role` is left out.", "type": "string" } ] }, "RotationConfig": { "description": "When a file is closed and the next one started.\n\nBoth triggers are optional and are checked together — whichever comes first\nrotates. With neither, a pipeline writes one file for as long as it runs.\n\nShared with the object-store output rather than local-only: \"how big does a\npart get\" is the same question on a disk and in a bucket, and the answer\nbelongs in one place.", "properties": { "interval_secs": { "description": "close the file this many seconds after it was opened. Measured from the\nopen, not from the last write, so files line up on a predictable cadence.", "format": "uint64", "minimum": 0, "type": [ "integer", "null" ] }, "max_rows": { "description": "close the file once it holds this many messages", "format": "uint", "minimum": 0, "type": [ "integer", "null" ] } }, "type": "object" }, "RunStatus": { "description": "Where a pipeline's run loop has got to.\n\nA pipeline is a spawned task, and until this existed nothing could say\nwhether that task was still alive: a run loop that had ended left its\nhandle in the map looking exactly like a running one — same card, same\nconfig, same everything, and no messages ever again. That is the zombie\nthis names.\n\nDeliberately four states and not a `bool`. \"Not running\" has three causes\nthat want different reactions: one is waiting for a database to come back\nand needs nothing done, one is a graph being torn down, and one is a\npipeline that is over.", "oneOf": [ { "const": "starting", "description": "Spawned, with an output still being initialised. A pipeline whose\ndatabase is not up yet sits here — retrying on a backoff — rather than\ndying, and leaves on its own the moment the far end answers.", "type": "string" }, { "const": "running", "description": "Initialised and in the loop. The only state in which messages move.", "type": "string" }, { "const": "stopped", "description": "The loop ended because it was cancelled: a delete, a revert or a\nshutdown. Rarely seen, because the handle is normally dropped with it.", "type": "string" }, { "const": "failed", "description": "The loop ended on its own — the last input died. Nothing will come out\nof this pipeline again until something rebuilds it.", "type": "string" } ] }, "S3Connection": { "description": "A bucket on an S3-compatible object store, and the credentials that reach\nit.\n\nThe `bucket` is where [`FileConnection`]'s `root` is: the thing the *system*\ngives you, against which an output names a prefix of its own. What is not\nhere is any equivalent of `--data-dir`. There cannot be one — the server has\nno view of a remote namespace to confine writes within, so the boundary is\nthe credentials, and giving a deployment a key that can only write one bucket\nis the thing that does what the sandbox does locally.\n\n`endpoint` is what makes this work against rustfs, minio or any other\nS3-compatible server; left out, it is real AWS S3 in `region`.", "properties": { "access_key_id": { "$ref": "#/components/schemas/Secret", "description": "access key id. May reference secrets as `${NAME}` — see \"secrets\" in the\nreadme, and prefer a reference to a literal here." }, "allow_http": { "description": "allow a plaintext `http://` endpoint. Defaults to false: credentials\nover http is a mistake worth having to write down, and the local rustfs\nis the case that legitimately wants it.", "type": [ "boolean", "null" ] }, "bucket": { "description": "the bucket to write into. It has to exist already — an output creates\nobjects, never buckets.", "type": "string" }, "endpoint": { "description": "url of an S3-compatible server, e.g. `http://localhost:9000` for the\nrustfs in `docker-compose.yaml`. Leave it out for real AWS S3, which is\nthen addressed through `region`.", "type": [ "string", "null" ] }, "region": { "description": "the bucket's region. Defaults to `us-east-1`, which is also what an\nS3-compatible server that does not care about regions will accept.", "type": [ "string", "null" ] }, "secret_access_key": { "$ref": "#/components/schemas/Secret", "description": "secret access key. May reference secrets as `${NAME}` — see \"secrets\" in\nthe readme, and prefer a reference to a literal here." } }, "required": [ "bucket", "access_key_id", "secret_access_key" ], "title": "s3", "type": "object" }, "S3OutputConfig": { "description": "Writes each batch to objects under a prefix in an S3-compatible bucket.\n\nThe same writer as the `file` output — the same part naming, the same\nformats, the same rotation policy — pointed at a bucket instead of a\ndirectory. What differs is that an object store has no append: a part is\nbuffered in memory and uploaded whole when it rotates, so `rotate` is\n**required** here and is what decides both how often objects appear and how\nmuch a running pipeline holds.", "properties": { "connection": { "description": "name of the s3 connection to write through — see \"connections\" in the\nreadme. The bucket and credentials live there; the prefix below is this\noutput's own.", "type": "string", "x-connection": "s3" }, "format": { "anyOf": [ { "$ref": "#/components/schemas/FileFormat" }, { "type": "null" } ], "description": "how the messages are laid out. Defaults to `ndjson`." }, "prefix": { "description": "key prefix to write under, e.g. `orders` — objects land at\n`/`. Leave it empty to write at the root of\nthe bucket.", "type": "string" }, "rotate": { "$ref": "#/components/schemas/RotationConfig", "description": "when to finish an object and start the next one. Required: an object\nstore cannot be appended to, so without a rotation trigger a pipeline\nwould hold its entire run in memory and upload it once, at the end." } }, "required": [ "connection", "prefix", "rotate" ], "title": "s3", "type": "object" }, "SampleRequest": { "description": "Take a few messages from an input, without creating a pipeline.", "properties": { "input": { "$ref": "#/components/schemas/InputConfig", "description": "the input to read from, exactly as it would be written in a pipeline —\nincluding its `envelope`, so the metadata fields the messages will\nreally carry are in the sample too." }, "max_messages": { "description": "how many messages to take. Defaults to [`DEFAULT_MAX_MESSAGES`], capped\nat [`MAX_MESSAGES`].", "format": "uint", "minimum": 0, "type": [ "integer", "null" ] }, "timeout_ms": { "description": "how long to wait for them, in milliseconds. Defaults to\n[`DEFAULT_TIMEOUT_MS`], capped at [`MAX_TIMEOUT_MS`]. The wait ends as\nsoon as enough messages have arrived.", "format": "uint64", "minimum": 0, "type": [ "integer", "null" ] } }, "required": [ "input" ], "title": "sample request", "type": "object" }, "SampleResponse": { "description": "What a sample found.\n\nNote there is no separate \"nothing arrived\" arm: an empty `messages` is\nexactly that, and it is an ordinary answer rather than a failure. A stream\nnobody is publishing to is a real state of the world, and one the user\noften needs to be shown rather than protected from.", "oneOf": [ { "properties": { "messages": { "description": "the messages, exactly as the pipeline's first transform would have\nseen them.", "items": true, "type": "array" }, "notes": { "description": "what the sample did differently from the pipeline it is standing in\nfor — a throwaway consumer group, an ignored buffer. Empty when it\ndid nothing differently.", "items": { "type": "string" }, "type": "array" }, "outcome": { "const": "sampled", "type": "string" }, "waited_ms": { "description": "how long it waited, in milliseconds.", "format": "uint64", "minimum": 0, "type": "integer" } }, "required": [ "outcome", "messages", "waited_ms" ], "type": "object" }, { "properties": { "message": { "type": "string" }, "outcome": { "const": "failed", "type": "string" }, "stage": { "$ref": "#/components/schemas/SampleStage" } }, "required": [ "outcome", "stage", "message" ], "type": "object" } ], "title": "sample response" }, "SampleStage": { "description": "Where a sample went wrong, when it did.", "oneOf": [ { "const": "build", "description": "The input could not be built at all — a connection that isn't\nconfigured, a secret that doesn't resolve. The same failure creating\nthe pipeline would have given, which is the point of building the real\nthing.", "type": "string" }, { "const": "read", "description": "It was built, and reading from it failed — the broker refused the\nsubscription, the host is unreachable.", "type": "string" } ] }, "SaveConfigRequest": { "description": "What `POST /api/config/save` takes: a bare file name, saved beside the\nconfig the server was started from. Not a path — see `persist::save_path`.", "properties": { "format": { "anyOf": [ { "$ref": "#/components/schemas/ConfigFormat" }, { "type": "null" } ], "default": null, "description": "JSON or YAML. Omitted means \"whatever `name` says it is\", which is what\nkeeps a client that predates the choice — and a hand-written `curl` —\nwriting the format the file is named for." }, "name": { "type": "string" }, "overwrite": { "default": true, "description": "Whether an existing file under this name may be replaced. `false` turns\nthe save into a *create*: a name that already exists is refused with a\n409 and nothing is written. The project creator sends `false`, because\nits user has typed a suggested default into a directory they have never\nseen — one Enter keypress away from replacing a config nobody meant to\ntouch.\n\nDefaults to `true`, because omitted has to stay byte-for-byte the old\nbehaviour — \"save as\" over the loaded file's own name is how a save has\nalways overwritten it, and a client that predates this field must keep\nworking.", "type": "boolean" } }, "required": [ "name" ], "title": "SaveConfigRequest", "type": "object" }, "SaveConfigResponse": { "description": "Where a save actually landed, so the UI can name it rather than guess.", "properties": { "path": { "type": "string" } }, "required": [ "path" ], "title": "SaveConfigResponse", "type": "object" }, "ScriptScope": { "description": "Whether a script is handed one message or the whole batch.\n\n`message` is the default and is what nearly everything wants: the budget is\nthen spent per message rather than per batch, the batch structure is\npreserved without the script having to rebuild it, and a script that emits\nnothing for one message has dropped exactly that message.\n\n`batch` is the escape hatch, and it is needed for the things that are about\nthe batch itself — deduplicating within it, repartitioning it, or computing\nsomething across it that `reduce` has no function for.", "oneOf": [ { "const": "message", "description": "The script runs once per message, with the message in `msg`.", "type": "string" }, { "const": "batch", "description": "The script runs once per batch, with the messages in `batch` as an\narray. Emitting an array emits a batch of those messages.", "type": "string" } ] }, "ScriptSource": { "description": "Where the script's text comes from.\n\nTwo spellings because the three ways someone writes a pipeline want\ndifferent things. Inline is what the HTTP API and the UI can carry — a\nscript in a file is a reference the browser cannot edit and a generated\nconfig has nowhere to put — and YAML renders it as a literal block, so it\nreads as code rather than as an escaped string. A file is what an editor can\nsyntax-highlight, a formatter can format and a test can exercise on its own,\nwhich is what the file-first workflow wants.\n\nInline is the canonical form: a `file` is resolved when the pipeline is\nbuilt and the config keeps the reference, so saving never inlines someone's\nfile out of existence.", "oneOf": [ { "description": "The script's text, in the config itself. Prefer a YAML config for this —\na literal block keeps it readable, where JSON has to escape every\nnewline.", "properties": { "code": { "description": "the rhai source", "type": "string", "x-script": "rhai" }, "type": { "const": "inline", "type": "string" } }, "required": [ "type", "code" ], "type": "object" }, { "description": "A path to a `.rhai` file, relative to the directory the config file is\nin — the same place the connections and layout files live.\n\nThe file is read when the pipeline is built — as are any modules it\n`import`s, which resolve against the same directory — so editing one\ntakes a revert to pick up. A server running without a config file has\nno directory to resolve against and refuses this; inline scripts still\nwork there, though their imports are refused for the same reason.", "properties": { "path": { "description": "the path, relative to the config file's directory. It may not climb\nout of that directory.", "type": "string" }, "type": { "const": "file", "type": "string" } }, "required": [ "type", "path" ], "type": "object" } ] }, "ScriptTransformConfig": { "description": "Runs a [rhai](https://rhai.rs) script over each message, or over the batch\nas a whole, and emits whatever the script asks for.\n\nA script reaches the message as `msg`, and emits with `emit(value)` — zero\ntimes to drop it, once to replace it, many times to split it. That covers\n`filter`, `map` and `splitter` in one, which is the point: what a script is\nfor is the case none of those three reach.\n\nThe script is **compiled when the pipeline is built**, so a syntax error is\na pipeline that refuses to start rather than one that fails every batch\nforever — the same rule the reducer's build-time checks follow. What cannot\nbe checked until a message arrives (a field that isn't there, a type that\nwon't convert) fails that batch and shows up on the card.\n\nEvery script runs under an **operation budget**. That is not a tuning knob\nwith a safe default, it is what makes this component safe to have: the\nscript runs synchronously inside the run loop's task, so a script that loops\nforever would wedge a worker thread rather than merely breaking its own\npipeline.\n\nA script may **`import`** other rhai files — shared helpers, written once —\nby a literal path relative to the config file's directory, which it may not\nclimb out of; the `.rhai` extension is implied. Imports resolve when the\npipeline is built, so a broken one refuses to start rather than failing\nbatches, and a running script never touches the filesystem.", "properties": { "max_operations": { "description": "how many rhai operations one run of the script may take before it is\nstopped and the batch failed. Leave it out for the default, which is\ngenerous for anything that isn't looping by mistake; raise it for a\nscript that legitimately walks a large array.", "format": "uint64", "minimum": 0, "type": [ "integer", "null" ] }, "scope": { "$ref": "#/components/schemas/ScriptScope", "description": "whether the script sees one message at a time or the whole batch" }, "source": { "$ref": "#/components/schemas/ScriptSource", "description": "the script itself, written inline or kept in a file beside the config" } }, "required": [ "source" ], "title": "script", "type": "object" }, "Secret": { "description": "A config value that may *reference* secrets rather than contain them.\n\nOn the wire it is an ordinary JSON string, but `${NAME}` placeholders in it\nare replaced with real values when the pipeline is built, against whatever\nsecret store the server was started with:\n\n```json\n{ \"type\": \"nats\", \"urls\": \"nats://app:${NATS_PASSWORD}@broker:4222\" }\n```\n\nThe unresolved form is the only one this type ever holds. That is what makes\nit safe to commit, safe to hand back from `GET /api/pipelines` and safe to show\nin the UI — a resolved value exists only inside the built runtime component,\nnever in a `Config`. Resolution deliberately lives in the root crate: this\ncrate compiles to wasm for the frontend, which must not be able to hold a\nresolved secret at all.\n\nA value with no `${...}` in it is passed through untouched, so fields that\nhold nothing sensitive need no special handling.", "type": "string" }, "SettingsDto": { "description": "How the server was started, and whether what it is running still matches\nthe file it started from.", "properties": { "config_file": { "description": "Name of the config file the server is working against — the `--config`\none, or the one a save has since created. Its absence doesn't mean edits\ncan't be saved: it means there is no file *yet*, so the UI offers to\ncreate one rather than to overwrite one.", "type": [ "string", "null" ] }, "save_directory": { "default": "", "description": "The directory a save writes into. Shown so \"create a config file\" can\nsay where the file will appear, which is the one thing the file name on\nits own doesn't tell you.\n\nDefaults to empty when a client is talking to an older server, which\nreads the same as \"unknown\" — the UI just leaves the location out.", "type": "string" }, "unsaved_changes": { "description": "The running graph has diverged from what was last loaded or saved.\nEdits apply to the runtime immediately and the file is left alone, so\nwithout this the divergence would be invisible until a restart lost it.", "type": "boolean" } }, "required": [ "unsaved_changes" ], "title": "SettingsDto", "type": "object" }, "Side": { "description": "A face of a card. Which one an edge uses is worked out from where the two\ncards sit and is never stored — but *where along it* the edge attaches can\nbe, and a stored position only means anything together with the face it was\nmeasured on.", "enum": [ "top", "right", "bottom", "left" ], "type": "string" }, "SplitterTransformConfig": { "description": "Cuts one batch into several smaller ones — the opposite of `buffer`.\n\nNote the current limitation: messages left over after the last whole chunk\nare dropped, so 4 messages with `out_size: 3` emit one batch, not two.", "properties": { "out_size": { "description": "how many messages go in each emitted batch", "format": "uint", "minimum": 0, "type": "integer" } }, "required": [ "out_size" ], "title": "splitter", "type": "object" }, "Stage": { "description": "The stage of a run loop an event came from. Also what the frontend matches\non to decide whether an edge lights up and which badge a log line gets, so\nit is a type rather than a string: both ends match on it exhaustively, and a\nfourth stage would fail to compile at every site that has to handle it.\n\nThe serialized spellings are wire format — `/events` carries them and the\nfrontend's filter chips are named after them. `stage_round_trips` pins them.", "enum": [ "input", "transform", "output" ], "type": "string" }, "StageResult": { "description": "What one transform in the chain did.", "properties": { "batches": { "description": "what it handed on, one entry per batch. Several batches is a\n`splitter`; none is a `filter` that dropped everything, or a `buffer`\nstill holding what it was given.", "items": { "items": true, "type": "array" }, "type": "array" }, "index": { "description": "its position in `transforms`.", "format": "uint", "minimum": 0, "type": "integer" }, "kind": { "description": "which transform it is (`filter`, `reduce`, ...), so the answer can be\nread without counting down the list.", "type": "string" }, "on_flush": { "description": "what it handed on when the chain was drained at the end, if anything.\n\nKept apart from `batches` because the difference is the interesting\npart: a running pipeline releases these on a timer or a gate rather\nthan when the messages arrive, so a dry run that folded the two\ntogether would make a `buffer` look like it passes everything straight\nthrough.", "items": { "items": true, "type": "array" }, "type": "array" } }, "required": [ "index", "kind", "batches" ], "type": "object" }, "StdoutOutputConfig": { "description": "Prints each batch to the server's stdout. Useful while building a pipeline\nup; takes no settings.", "title": "stdout", "type": "object" }, "StringFilterOperatorKind": { "description": "How a string is compared to the one in the config.", "enum": [ "equal_to", "contains" ], "type": "string" }, "TableIndex": { "description": "An index to create alongside the table.\n\nOnly created when the table is — like the table itself it is\n`IF NOT EXISTS`, and an index on a table someone else owns is theirs to\nmanage.", "properties": { "columns": { "description": "the columns to index, in order. Each must be one of the mapped columns.", "items": { "type": "string" }, "type": "array" }, "unique": { "description": "whether the index is unique. Defaults to false.", "type": [ "boolean", "null" ] } }, "required": [ "columns" ], "title": "index", "type": "object" }, "TokenLoginRequest": { "description": "What `POST /api/auth/token` takes.\n\nThe token is the host application's — minted by its identity provider,\ncarried here from the embedding page's URL. Like [`LoginRequest`]'s\npassword it is a live credential rather than a `${NAME}` reference: it\nexists for the length of one request, is exchanged for a session cookie,\nand is never stored, serialized back or logged.", "properties": { "token": { "type": "string" } }, "required": [ "token" ], "title": "TokenLoginRequest", "type": "object" }, "TransformConfig": { "oneOf": [ { "$ref": "#/components/schemas/BufferTransformConfig", "properties": { "type": { "const": "buffer", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/HttpTransformConfig", "properties": { "type": { "const": "http", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/SplitterTransformConfig", "properties": { "type": { "const": "splitter", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/ReduceTransformConfig", "properties": { "type": { "const": "reducer", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/FilterTransformConfig", "properties": { "type": { "const": "filter", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/RememberTransformConfig", "properties": { "type": { "const": "remember", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/RecallTransformConfig", "properties": { "type": { "const": "recall", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/MapTransformConfig", "properties": { "type": { "const": "map", "type": "string" } }, "required": [ "type" ], "type": "object" }, { "$ref": "#/components/schemas/ScriptTransformConfig", "properties": { "type": { "const": "script", "type": "string" } }, "required": [ "type" ], "type": "object" } ], "type": "object" }, "UiEvent": { "properties": { "component": { "default": null, "description": "Which component of the stage, indexed into that stage's array in the\nconfig — the second of two outputs is `Some(1)`.\n\n`None` where it isn't known rather than where there is only one: input\nbatches carry no index because several inputs are merged before the run\nloop sees them, and by then which one produced the batch is gone.", "format": "uint", "minimum": 0, "type": [ "integer", "null" ] }, "payload": { "$ref": "#/components/schemas/EventPayload" }, "pipeline_id": { "type": "string" }, "seq": { "default": null, "description": "Which pass through the run loop this belongs to — one batch in, its\ntransforms, and everything that left. Counted per pipeline from one.\n\n`None` for anything that happened outside a pass: an output that failed\nto initialise before the loop started, or an input source dying in its\nown task while the loop waits. Those are real events with no pass to\nbelong to, not a missing number.\n\nThe frontend groups the log by this, and a *gap* in it is information\ntoo: the UI feed is a broadcast channel that drops rather than blocks,\nso a jump from 8 to 12 is three passes the browser never saw and should\nsay so instead of drawing the survivors as if they were consecutive.", "format": "uint64", "minimum": 0, "type": [ "integer", "null" ] }, "stage": { "$ref": "#/components/schemas/Stage" }, "ts": { "default": 0, "description": "When the run loop reported this, in milliseconds since the epoch.\n\nThe *server's* clock, stamped where the event is published rather than\nwhere it is built: this type compiles for wasm, where `SystemTime::now`\npanics. Zero means \"no time\" — an event from a server that predates the\nfield, which the log renders as blank rather than as 1970.", "format": "uint64", "minimum": 0, "type": "integer" } }, "required": [ "pipeline_id", "stage", "payload" ], "title": "UiEvent", "type": "object" }, "UnionDoc": { "description": "A tagged union as a form can render it: pick the tag, then fill in whatever\nthat variant asks for.", "properties": { "tag": { "description": "The property that says which variant this is — `type`, for every union\nin the config today. Carried rather than assumed, because it is what\ngoes on the wire beside the variant's own fields.", "type": "string" }, "variants": { "description": "The variants, named by their tag *value* (`static`, `tumbling`) rather\nthan by a Rust variant name. The tag itself is not among any variant's\nfields — it is the choice, not a thing to fill in.", "items": { "$ref": "#/components/schemas/VariantDoc" }, "type": "array" } }, "required": [ "tag", "variants" ], "type": "object" }, "VariantDoc": { "description": "A component config that is a tagged enum rather than a flat struct — the\n`filter` transform, whose fields depend on which kind of filter it is.", "properties": { "fields": { "items": { "$ref": "#/components/schemas/FieldDoc" }, "type": "array" }, "name": { "type": "string" } }, "required": [ "name", "fields" ], "type": "object" } }, "securitySchemes": { "basicAuth": { "description": "HTTP Basic credentials, checked against the accounts in the server's `--server-config` file. This is what anything that is not a browser should use.", "scheme": "basic", "type": "http" }, "cookieAuth": { "description": "A session cookie from `POST /api/auth/login`. It exists for browsers: `EventSource`, which `GET /events` is consumed with, cannot send headers, so Basic credentials are not available on the one endpoint the UI needs most.", "in": "cookie", "name": "kayak_session", "type": "apiKey" } } }, "info": { "description": "Graph-based stream processing: an HTTP API over configurable `input → transforms → output` pipelines.\n\nEdits through this API apply to the running graph immediately and write nothing to disk — the config file is a load source and a save target, never a mirror of the runtime. `POST /api/config/save` is the only thing that writes it, and `POST /api/config/revert` is the only undo.", "title": "kayak", "version": "0.1.2" }, "openapi": "3.1.0", "paths": { "/api/auth/login": { "post": { "description": "Checks a username and password against the accounts in the server's settings file and, on success, sets an `HttpOnly` session cookie.\n\nThis is for browsers. Everything else should send `Authorization: Basic` on each request instead and never come here — the cookie exists because `EventSource`, which the UI consumes `/events` with, cannot send headers.\n\nA wrong password and an unknown username are the same 401, deliberately: the endpoint is not a way to find out who has an account. On a server with no accounts configured this is not an error either — it answers 200 with `authentication_required` false, because there is nothing to sign into.", "operationId": "login", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/LoginRequest" } } }, "description": "The credentials to check.", "required": true }, "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AuthDto" } } }, "description": "Signed in. The session cookie is in `Set-Cookie`." }, "401": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Wrong username or password — the body does not say which." }, "500": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Something went wrong on the server. The body says what." } }, "security": [], "summary": "Exchange credentials for a session", "tags": [ "auth" ], "x-kayak-access": "public" } }, "/api/auth/logout": { "post": { "description": "Clears the cookie in the browser and drops the session on the server, so a copy of the cookie taken from somewhere else stops working too.\n\nIdempotent: 204 whether or not there was a session to end.", "operationId": "logout", "responses": { "204": { "description": "Signed out." }, "401": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "No credentials, or credentials that were not recognised." } }, "security": [ { "basicAuth": [] }, { "cookieAuth": [] } ], "summary": "End the session this request carries", "tags": [ "auth" ], "x-kayak-access": "read" } }, "/api/auth/me": { "get": { "description": "`authentication_required` says whether this server checks credentials at all — a server started without a `--server-config`, or with one declaring `auth: {type: none}`, answers `false` and lets everybody do everything.\n\n`username` and `role` describe the caller, and are both null for one who presented nothing. Note that a null `role` is not the same as `read`: a reader may see the graph, a signed-out caller may not.\n\nCallable without credentials, necessarily — it is the endpoint that answers 'do I need to show a login page'.", "operationId": "whoAmI", "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AuthDto" } } }, "description": "Who you are. Not an error even when the answer is nobody." } }, "security": [], "summary": "Who the caller is, and whether this server asks", "tags": [ "auth" ], "x-kayak-access": "public" } }, "/api/auth/token": { "post": { "description": "The embedding flow's endpoint, on a server whose auth section is `jwt`: a host application that already holds a token from the shared identity provider — Cognito, Keycloak — puts it on the iframe URL as `?auth_token=`, and the UI posts it here once. The token is checked against the issuer's published keys and, on success, exchanged for the same `HttpOnly` session cookie a password login sets — so the token itself appears in exactly one request and never in an access log again.\n\nThe session ends no later than the token's `exp`: the cookie must not outlive the identity provider's word that the caller is signed in.\n\nAPI callers don't need this exchange — on a `jwt` server, `Authorization: Bearer ` works directly on every endpoint.\n\nEvery way of being refused is the same 401, deliberately: an expired token, a wrong issuer and a server that doesn't take tokens at all are not distinctions worth handing to a guesser.", "operationId": "tokenLogin", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TokenLoginRequest" } } }, "description": "The token to check.", "required": true }, "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AuthDto" } } }, "description": "Signed in. The session cookie is in `Set-Cookie`." }, "401": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "The token was not accepted, or this server does not take tokens." }, "500": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Something went wrong on the server. The body says what." } }, "security": [], "summary": "Exchange an identity provider's JWT for a session", "tags": [ "auth" ], "x-kayak-access": "public" } }, "/api/config/revert": { "post": { "description": "The undo for a session of editing, and as destructive as it sounds: every running pipeline is stopped and the graph is rebuilt from the file.\n\nThe file is parsed *before* the runtime is torn down, so a file broken by hand costs you nothing. The connections are reloaded first, since the pipelines being rebuilt name them. It waits for the old pipelines to actually stop before rebuilding, so the response landing means the new graph is the only one running.", "operationId": "revertConfig", "responses": { "204": { "description": "Reloaded; the graph is what the file says." }, "401": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "No credentials, or credentials that were not recognised." }, "403": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Signed in, but without the `admin` role this endpoint needs." }, "500": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "There is no config file to revert to, or it could not be read or parsed — in which case the running graph is left alone." } }, "security": [ { "basicAuth": [] }, { "cookieAuth": [] } ], "summary": "Throw the running graph away and reload the config file", "tags": [ "config" ], "x-kayak-access": "admin" } }, "/api/config/save": { "post": { "description": "Writes the running pipelines out in a deterministic order (topological, ties by id) via a temp file and a rename, because the result is meant to be committed. The connections file and the canvas layout are written beside it by the same save — a config saved without the connections it names would not start.\n\n`name` is a **bare file name**, not a path, and is validated as one: the file lands in the server's save directory and nowhere else. Using the loaded file's own name is how you overwrite it.\n\n`format` picks JSON or YAML; leaving it out takes the format from the name's extension. On a server started without `--config` this is how a config file comes into existence at all, and from that save on it is the file `revert` reloads.\n\n`overwrite` defaults to `true`, which is what makes saving over the loaded file the ordinary thing it has always been. Sending `false` turns the save into a **create**: if the name — or either of the two files written beside it — is already on disk, the request is refused with a 409 and nothing is written. That is what the UI's project creator sends, since it suggests a file name into a directory its user has often never looked at.", "operationId": "saveConfig", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SaveConfigRequest" } } }, "description": "The file name to write, optionally the format, and whether an existing file may be replaced.", "required": true }, "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SaveConfigResponse" } } }, "description": "Written, with the path it landed at." }, "401": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "No credentials, or credentials that were not recognised." }, "403": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Signed in, but without the `admin` role this endpoint needs." }, "409": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "`overwrite` was `false` and the file — or one of the two written beside it — is already there. Nothing was written; the message names the files." }, "422": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "`name` is not a bare file name." }, "500": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Something went wrong on the server. The body says what." } }, "security": [ { "basicAuth": [] }, { "cookieAuth": [] } ], "summary": "Write the running graph to a config file", "tags": [ "config" ], "x-kayak-access": "admin" } }, "/api/connections": { "get": { "description": "Keyed by name, which is the same shape as the connections file itself — what the UI lists and what gets committed are one thing, so there is no second format to keep in step.\n\nCredentials come back as the unresolved `${NAME}` templates they are configured as, never as their values.", "operationId": "listConnections", "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Connections" } } }, "description": "Every configured connection, in name order." }, "401": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "No credentials, or credentials that were not recognised." } }, "security": [ { "basicAuth": [] }, { "cookieAuth": [] } ], "summary": "The connections pipelines can name", "tags": [ "connections" ], "x-kayak-access": "read" }, "post": { "description": "Changes what the *next* pipeline build can name, and nothing else: a component reads its connection once, when it is built, so editing or adding one reaches only new and rebuilt pipelines.\n\nLike creating a pipeline this writes nothing to disk; the save does, and it writes the config and the connections file together.", "operationId": "createConnection", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CreateConnectionRequest" } } }, "description": "The connection, and the name to file it under.", "required": true }, "responses": { "201": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CreateConnectionRequest" } } }, "description": "Added, echoed back as stored." }, "401": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "No credentials, or credentials that were not recognised." }, "403": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Signed in, but without the `admin` role this endpoint needs." }, "409": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "A connection of that name already exists." }, "500": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Something went wrong on the server. The body says what." } }, "security": [ { "basicAuth": [] }, { "cookieAuth": [] } ], "summary": "Add a connection", "tags": [ "connections" ], "x-kayak-access": "admin" } }, "/api/connections/{connection_id}": { "delete": { "description": "Refused while a running pipeline still names it — that comes back as a 409 listing the pipelines, so the answer says what to do about it.", "operationId": "deleteConnection", "parameters": [ { "description": "The name the connection is filed under.", "in": "path", "name": "connection_id", "required": true, "schema": { "type": "string" } } ], "responses": { "204": { "description": "Removed." }, "401": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "No credentials, or credentials that were not recognised." }, "403": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Signed in, but without the `admin` role this endpoint needs." }, "404": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "No connection of that name exists." }, "409": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Running pipelines still name it; the body lists them." }, "500": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Something went wrong on the server. The body says what." } }, "security": [ { "basicAuth": [] }, { "cookieAuth": [] } ], "summary": "Remove a connection", "tags": [ "connections" ], "x-kayak-access": "admin" } }, "/api/docs": { "get": { "description": "Every input, transform, output and connection kayak can build, with their fields, types and documentation — reflected out of the config schemas, so it cannot drift from what the server actually accepts.\n\nThe `/docs` *page* generates the same thing in the browser from the same code, so this endpoint isn't what renders it. It exists because the component reference is useful to things that aren't a browser: a config linter, editor completion, a test.", "operationId": "listComponents", "responses": { "200": { "content": { "application/json": { "schema": { "items": { "$ref": "#/components/schemas/ComponentDoc" }, "type": "array" } } }, "description": "Every component, grouped by nothing — `family` says which plugin point each one plugs into." } }, "security": [], "summary": "The component reference, as data", "tags": [ "reference" ], "x-kayak-access": "public" } }, "/api/inputs/sample": { "post": { "description": "Configuring a stream you cannot see is guesswork, and every field reference downstream — a column's `field`, a filter's comparison — is a name someone had to already know. This builds the input in the body exactly as a pipeline would, takes up to `max_messages` from it within `timeout_ms`, and drops it.\n\nThe **real** input, including its `envelope`, so the metadata fields the messages will actually carry are in the sample too. Its `buffer` is the one thing ignored: a buffer's job is to make the pipeline wait, which is not what a sample is for. Anything the sample did differently comes back in `notes`.\n\n**Sampling is not free for every kind of input, and the ones where it isn't say so.** A kafka sample runs under a throwaway consumer group, so it neither rebalances the pipeline's group nor commits on its behalf; an mqtt sample connects under a client id of its own, because a broker disconnects the older client holding one. An `http` input is refused outright with a 400 — it is posted to rather than read from, so there is nothing to fetch.\n\n**No messages is a 200 with an empty list.** A subject nobody is publishing to is a real state of the world and the answer to the question asked; none of these inputs can replay what was published before the sample started.", "operationId": "sampleInput", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SampleRequest" } } }, "description": "The input to read from, and how much to take.", "required": true }, "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SampleResponse" } } }, "description": "The sample was taken — `outcome` says whether it produced messages or failed on the way." }, "400": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "The request itself was wrong: malformed JSON, an input that isn't a kind of input, or one that cannot be sampled at all." }, "401": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "No credentials, or credentials that were not recognised." }, "403": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Signed in, but without the `admin` role this endpoint needs." }, "500": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Something went wrong on the server. The body says what." } }, "security": [ { "basicAuth": [] }, { "cookieAuth": [] } ], "summary": "Fetch a few real messages from an input, without creating a pipeline", "tags": [ "pipelines" ], "x-kayak-access": "admin" } }, "/api/layout": { "get": { "description": "Served separately from `/api/pipelines` because it is a different kind of thing: that is what the server is running, this is how someone chose to look at it. A client that ignores this endpoint gets an automatically laid out graph, which is the point.\n\nOnly pipelines someone has actually moved appear.", "operationId": "getLayout", "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/LayoutFile" } } }, "description": "The stored arrangement." }, "401": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "No credentials, or credentials that were not recognised." } }, "security": [ { "basicAuth": [] }, { "cookieAuth": [] } ], "summary": "Where the cards sit on the canvas", "tags": [ "layout" ], "x-kayak-access": "read" }, "put": { "description": "The whole map, not a patch: the canvas already holds the complete arrangement, and a full replacement is what makes \"reset everything to automatic\" a send of `{}` rather than its own endpoint.\n\nThis is the one edit that writes immediately rather than waiting for a save, and it never counts as an unsaved change — moving a card changes nothing the server runs, so there is nothing worth reviewing before it lands. Without a config file there is nowhere to write, and the arrangement is kept in memory until a save creates one.", "operationId": "replaceLayout", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/LayoutFile" } } }, "description": "The complete arrangement.", "required": true }, "responses": { "204": { "description": "Stored, and written if there is a file to write." }, "401": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "No credentials, or credentials that were not recognised." }, "403": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Signed in, but without the `admin` role this endpoint needs." }, "500": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Something went wrong on the server. The body says what." } }, "security": [ { "basicAuth": [] }, { "cookieAuth": [] } ], "summary": "Replace the arrangement and write it to disk", "tags": [ "layout" ], "x-kayak-access": "admin" } }, "/api/openapi.json": { "get": { "description": "Generated from the same table the routes are registered from, with schemas reflected out of the Rust types — so it describes the server that is serving it.\n\nPoint a renderer, a client generator or a contract test at it. `GET /api/reference` is one such renderer, served alongside.", "operationId": "getOpenApi", "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/OpenApiDocument" } } }, "description": "The OpenAPI document." } }, "security": [], "summary": "This API, as an OpenAPI 3.1 document", "tags": [ "reference" ], "x-kayak-access": "public" } }, "/api/pipelines": { "get": { "description": "The running graph, as the configs the pipelines were built from — each with the id it is actually running under, which is generated when the config omitted one.\n\nThis is the runtime's view rather than the file's: a pipeline created since startup is here and not in the config file, and `GET /api/settings` is what says whether the two have diverged.", "operationId": "listPipelines", "responses": { "200": { "content": { "application/json": { "schema": { "items": { "$ref": "#/components/schemas/PipelineDto" }, "type": "array" } } }, "description": "The running pipelines, in no particular order." }, "401": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "No credentials, or credentials that were not recognised." }, "500": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Something went wrong on the server. The body says what." } }, "security": [ { "basicAuth": [] }, { "cookieAuth": [] } ], "summary": "Every pipeline the server is running", "tags": [ "pipelines" ], "x-kayak-access": "read" }, "post": { "description": "The body is one pipeline's config, exactly as it would appear in the `pipelines` array of a config file. Omitting `id` generates a readable random one, which comes back in the response.\n\nThe pipeline is built and started before the response is sent, so a 201 means it is running — a component that could not be built (an unknown connection, an unresolved secret) is a 422 and nothing is started. Nothing is written to disk: the config file is a load source and a save target, never a mirror of the runtime.", "operationId": "createPipeline", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Config" } } }, "description": "The pipeline to build.", "required": true }, "responses": { "201": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PipelineDto" } } }, "description": "Built and running, with the id it took." }, "401": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "No credentials, or credentials that were not recognised." }, "403": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Signed in, but without the `admin` role this endpoint needs." }, "409": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "A pipeline with this id is already running." }, "422": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "The config is well-formed JSON but could not be built — an unknown connection, a missing secret, an upstream that does not exist." }, "500": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Something went wrong on the server. The body says what." } }, "security": [ { "basicAuth": [] }, { "cookieAuth": [] } ], "summary": "Build and start a pipeline", "tags": [ "pipelines" ], "x-kayak-access": "admin" } }, "/api/pipelines/dry-run": { "post": { "description": "What a `map` writes, what a `filter` drops, what a `reduce` collapses a batch to — questions the config cannot answer and one real message can. This builds the transforms in the body exactly as a pipeline would and puts the messages down the chain, reporting **what each stage handed on**.\n\nPer stage and as a list of batches, because that is where the answer usually is: a `splitter` hands on several batches, a `filter` that dropped everything hands on none, and a `buffer` hands on nothing because it is still holding what it was given. What a transform only releases when the chain is drained is reported separately, as `on_flush`, so a buffer doesn't look like it passes everything straight through. A transform that is *still* holding what it was given hands on nothing at all, and the chain says so rather than pretending the messages came through: a dry run has no tick to give a window that has thirty seconds left on it.\n\n**There are no outputs and there cannot be.** A dry run that emitted would be a pipeline; everything up to the outputs is a question about the data, and the outputs are the part that changes somebody else's system.\n\n**State is never live**, exactly as for a script dry run: the buckets are private to the request, seeded from `buckets` in the body, returned in the response and thrown away with it.\n\nA transform that cannot be built, or that fails on a message, is a 200 whose `outcome` is `failed` — the request was carried out and where it broke is the answer. The stages that completed first come back with it.", "operationId": "dryRunPipeline", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PipelineDryRunRequest" } } }, "description": "The messages, and the transforms to put them through.", "required": true }, "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PipelineDryRunResponse" } } }, "description": "The chain ran, or it broke on the way — the `outcome` field says which." }, "400": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "The request itself was wrong: malformed JSON, or a transform that isn't a kind of transform." }, "401": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "No credentials, or credentials that were not recognised." }, "403": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Signed in, but without the `admin` role this endpoint needs." }, "500": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Something went wrong on the server. The body says what." } }, "security": [ { "basicAuth": [] }, { "cookieAuth": [] } ], "summary": "Run a draft's transforms over some messages, without creating a pipeline", "tags": [ "pipelines" ], "x-kayak-access": "admin" } }, "/api/pipelines/{pipeline_id}": { "delete": { "description": "Cancels the pipeline's run loop and drops it from the graph. Pipelines downstream of it keep running and stop receiving from it.\n\nLike creating one, this writes nothing to disk.", "operationId": "deletePipeline", "parameters": [ { "description": "The id the pipeline is running under.", "in": "path", "name": "pipeline_id", "required": true, "schema": { "type": "string" } } ], "responses": { "204": { "description": "Stopped and removed." }, "401": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "No credentials, or credentials that were not recognised." }, "403": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Signed in, but without the `admin` role this endpoint needs." }, "404": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "No pipeline is running under that id." }, "500": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Something went wrong on the server. The body says what." } }, "security": [ { "basicAuth": [] }, { "cookieAuth": [] } ], "summary": "Stop and remove a pipeline", "tags": [ "pipelines" ], "x-kayak-access": "admin" } }, "/api/pipelines/{pipeline_id}/history": { "get": { "description": "Throughput and failures over time, kept in the server's memory so that something which broke overnight can still be read in the morning.\n\nThis is the counterpart to `/events`, not a replay of it. The event stream is a live sample: it is only produced while a browser is attached and it drops passes under load on purpose. History is fed by counters the run loop keeps regardless of who is watching, so it is complete in what it counts — and correspondingly it carries no message payloads at all, only counts, and failures aggregated to one entry per distinct message with a first-seen, a last-seen and a tally.\n\nBuckets are contiguous and oldest first, including empty ones: a run of zeroes is a pipeline that stopped, which is a different fact from a gap and is spelled differently.\n\nAn unknown or newly created pipeline answers with an empty history rather than a 404 — a pipeline that has not done anything yet is not an error. How much is kept is the `history.retention_secs` in the server config; when that is zero nothing is recorded and this always answers empty.", "operationId": "getPipelineHistory", "parameters": [ { "description": "Id of the pipeline.", "in": "path", "name": "pipeline_id", "required": true, "schema": { "type": "string" } }, { "description": "`coarse` (the default) — a minute a bucket, over the configured retention, which is the overnight record. `fine` — five seconds a bucket over the last half hour, which is what a card's live chart is backfilled from so it starts full rather than drawing itself over the next two minutes.", "in": "query", "name": "resolution", "required": false, "schema": { "type": "string" } } ], "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PipelineHistory" } } }, "description": "The pipeline's history at the resolution asked for." }, "401": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "No credentials, or credentials that were not recognised." } }, "security": [ { "basicAuth": [] }, { "cookieAuth": [] } ], "summary": "What a pipeline has been doing, after the fact", "tags": [ "pipelines" ], "x-kayak-access": "read" } }, "/api/pipelines/{pipeline_id}/messages": { "post": { "description": "The endpoint a pipeline's `http` input serves. Every pipeline with one has this path, derived from its id, and it exists for as long as the pipeline is running — this is how a system pushes data into kayak without a broker in between.\n\nThe body is one JSON message or an array of them; an array arrives as a single batch, so posting ten messages is one pass through the transforms rather than ten. There is no envelope and no schema: whatever is posted is what the transforms see.\n\nAccepted means queued, not processed. The batch is handed to the pipeline's run loop and the response is sent without waiting for the outputs, so a 202 says nothing about whether the data has landed anywhere.\n\nThis endpoint does not use the server's sign-in — it is a data plane, and a system pushing readings should not need an account that can rewrite the graph. Protecting it is the `http` input's own `auth` field: a token the sender repeats in a header, declared per pipeline. Without one the endpoint takes anything that reaches it, which is the default.", "operationId": "ingestMessages", "parameters": [ { "description": "The id of the pipeline to post to.", "in": "path", "name": "pipeline_id", "required": true, "schema": { "type": "string" } } ], "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/IngestRequest" } } }, "description": "One message, or an array of messages to deliver as one batch.", "required": true }, "responses": { "202": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/IngestResponse" } } }, "description": "Queued for the pipeline, with the number of messages taken." }, "401": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "The input has an `auth` and this post didn't satisfy it. This is the input's own credential, not the server's sign-in: an account on the server does not let you post, and the token does not let you do anything else." }, "404": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "No pipeline is running under that id, or the one that is has no `http` input to post to." }, "500": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Something went wrong on the server. The body says what." }, "503": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "The pipeline's queue is full — it is not reading as fast as this is being posted. Nothing was taken; send it again." } }, "security": [], "summary": "Post messages into a pipeline", "tags": [ "pipelines" ], "x-kayak-access": "public" } }, "/api/reference": { "get": { "description": "An HTML page rendering `/api/openapi.json`, with a request panel for trying endpoints against this server.\n\nThe `/docs` page in the UI covers the same endpoints in kayak's own furniture; this is the full reference, schemas and all.", "operationId": "apiReference", "responses": { "200": { "content": { "text/html": { "schema": { "type": "string" } } }, "description": "The reference page." } }, "security": [], "summary": "The rendered API reference", "tags": [ "reference" ], "x-kayak-access": "public" } }, "/api/scripts/dry-run": { "post": { "description": "A `script` transform is the one component whose configuration can be wrong in a way the config's *shape* cannot express: for every other component, a config that deserializes and builds does what it says, and for this one the interesting mistakes are all inside a string. This endpoint is where that string gets checked.\n\nIt compiles the script and runs it over the messages in the body, through the same runner and under the same operation budget and sandbox a running transform gets — a dry run that could disagree with production would be worse than none, because it would be trusted.\n\n**A script with a bug in it is a 200, not a 400.** The request was well formed and the server answered it completely; where the bug is *is* the answer. The response is a tagged union: `emitted` carries the batches, `failed` carries the message with a line and column an editor can point at. A 400 here means the request itself was wrong — malformed JSON, or a `file` source naming something unreadable.\n\nState is **never live**. The run gets a private bucket seeded from `state` in the body and thrown away afterwards, and what it holds at the end comes back in the response. Reading production state would make the answer depend on what the server happened to be doing; writing it would give a dry run side effects.", "operationId": "dryRunScript", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/DryRunRequest" } } }, "description": "The script, and the messages to run it over.", "required": true }, "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/DryRunResponse" } } }, "description": "The script ran, or it did not compile — the `outcome` field says which." }, "400": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "The request itself was wrong: malformed JSON, or a `file` source that could not be read." }, "401": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "No credentials, or credentials that were not recognised." }, "403": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Signed in, but without the `admin` role this endpoint needs." }, "500": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "Something went wrong on the server. The body says what." } }, "security": [ { "basicAuth": [] }, { "cookieAuth": [] } ], "summary": "Run a script over some messages, without creating a pipeline", "tags": [ "pipelines" ], "x-kayak-access": "admin" } }, "/api/settings": { "get": { "description": "Which config file the server is working against, where a save would land, and whether the running graph has diverged from what was last loaded or saved.\n\nThe absence of a config file doesn't mean edits can't be saved: it means there is no file *yet*, and a save creates one.", "operationId": "getSettings", "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SettingsDto" } } }, "description": "The server's configuration state." }, "401": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "No credentials, or credentials that were not recognised." } }, "security": [ { "basicAuth": [] }, { "cookieAuth": [] } ], "summary": "How the server was started, and whether it has drifted", "tags": [ "config" ], "x-kayak-access": "read" } }, "/api/state": { "get": { "description": "One entry per bucket declared under `state` in the config, in name order, with the number of keys it is currently holding and the bounds it is held to.\n\nBuckets are not created or deleted through the API — they are part of the graph's logic and live in the config file, so this family is read-only.", "operationId": "listStateBuckets", "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/BucketSummary" } } }, "description": "Every declared bucket, in name order." }, "401": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "No credentials, or credentials that were not recognised." } }, "security": [ { "basicAuth": [] }, { "cookieAuth": [] } ], "summary": "The state buckets and how full they are", "tags": [ "state" ], "x-kayak-access": "read" } }, "/api/state/{bucket}": { "get": { "description": "The keys and the values remembered under each, most recently written first — which is the order that makes a live bucket readable, since the key that just changed is the one worth seeing.\n\nCapped: a bucket may hold thousands of keys and this returns a page of them, with `truncated` saying so and `keys` giving the real total. It is a snapshot taken under the bucket's lock, so it is consistent with itself and stale the moment it is sent.", "operationId": "getStateBucket", "parameters": [ { "description": "Name of the bucket, as declared in the config.", "in": "path", "name": "bucket", "required": true, "schema": { "type": "string" } } ], "responses": { "200": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/BucketContents" } } }, "description": "The bucket's contents." }, "401": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "No credentials, or credentials that were not recognised." }, "404": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "No bucket of that name is declared." } }, "security": [ { "basicAuth": [] }, { "cookieAuth": [] } ], "summary": "What one bucket is holding", "tags": [ "state" ], "x-kayak-access": "read" } }, "/events": { "get": { "description": "A `text/event-stream` of `UiEvent`s: a batch arriving at a stage, or a failure handling one. Each SSE `data:` field is one event as JSON.\n\nIt is a broadcast that drops rather than blocks, so a slow consumer misses events instead of slowing the pipelines down — which is what `seq` is for, since a gap in it is the honest report of what was missed. Run loops only publish at all while somebody is listening.\n\nThe stream is explicitly a dev-tooling affordance rather than a durable feed, and is marked temporary in the source.", "operationId": "streamEvents", "responses": { "200": { "content": { "text/event-stream": { "schema": { "description": "A stream of SSE frames whose `data:` field is one `UiEvent` as JSON.", "type": "string" } } }, "description": "An event stream that stays open." }, "401": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ApiError" } } }, "description": "No credentials, or credentials that were not recognised." } }, "security": [ { "basicAuth": [] }, { "cookieAuth": [] } ], "summary": "What the pipelines are doing, as it happens", "tags": [ "events" ], "x-kayak-access": "read" } } }, "tags": [ { "description": "The running graph. Creating and deleting pipelines takes effect immediately and writes nothing to disk.", "name": "pipelines" }, { "description": "The systems pipelines talk to, named once and referred to by the components that use them.", "name": "connections" }, { "description": "What the pipelines remember between batches. Read-only: buckets are declared in the config and filled by `remember` transforms, so there is nothing here to write.", "name": "state" }, { "description": "The config file: how the server was started, writing the running graph out to it, and throwing the graph away to start again from it.", "name": "config" }, { "description": "Where the cards sit on the canvas. Not configuration — this is written to its own file, and immediately.", "name": "layout" }, { "description": "What the pipelines are doing, as it happens.", "name": "events" }, { "description": "Signing in and out. Present on every server; on one with no accounts configured they report that there is nothing to sign into.", "name": "auth" }, { "description": "The API describing itself.", "name": "reference" } ] }