# -*- coding: utf-8 -*-
from setuptools import setup

packages = \
['streams_explorer',
 'streams_explorer.api',
 'streams_explorer.api.dependencies',
 'streams_explorer.api.routes',
 'streams_explorer.core',
 'streams_explorer.core.extractor',
 'streams_explorer.core.extractor.default',
 'streams_explorer.core.services',
 'streams_explorer.models']

package_data = \
{'': ['*']}

install_requires = \
['cachetools>=4.2.2,<5.0.0',
 'confluent-kafka>=1.9.2,<2.0.0',
 'dynaconf>=3.1.7,<4.0.0',
 'fastapi-utils>=0.2.1,<0.3.0',
 'fastapi>=0.78.0,<0.79.0',
 'httpx>=0.23.1,<0.24.0',
 'kubernetes-asyncio>=23.6.0,<24.0.0',
 'loguru>=0.5.3,<0.6.0',
 'matplotlib>=3.4.3,<4.0.0',
 'networkx>=2.6.3,<3.0.0',
 'pydantic>=1.9.1,<2.0.0',
 'pygraphviz>=1.10,<2.0',
 'uvicorn[standard]>=0.20.0,<0.21.0']

entry_points = \
{'console_scripts': ['start = main:start']}

setup_kwargs = {
    'name': 'streams-explorer',
    'version': '2.2.1',
    'description': 'Explore Data Pipelines in Apache Kafka.',
    'long_description': '# Streams Explorer\n\n> Explore Data Pipelines in Apache Kafka.\n\n![streams-explorer](https://github.com/bakdata/streams-explorer/blob/main/screens/overview.png?raw=true)\n\n> **Note**\n> We are participating in the annual Hacktoberfest. If you\'re looking to contribute, please see our [open issues](https://github.com/bakdata/streams-explorer/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3Ahacktoberfest) and use the [standalone installation](#standalone) for development.\n\n## Contents\n\n- [Streams Explorer](#streams-explorer)\n  - [Features](#features)\n  - [Overview](#overview)\n  - [Installation](#installation)\n    - [Docker Compose](#docker-compose)\n    - [Deploying to Kubernetes cluster](#deploying-to-kubernetes-cluster)\n    - [Standalone](#standalone)\n      - [Backend](#backend)\n      - [Frontend](#frontend)\n  - [Configuration](#configuration)\n    - [Kafka](#kafka)\n    - [Kafka Connect](#kafka-connect)\n    - [Kubernetes](#kubernetes)\n    - [Schema Registry / Karapace](#schema-registry--karapace)\n    - [Prometheus](#prometheus)\n    - [AKHQ](#akhq)\n    - [Redpanda Console](#redpanda-console)\n    - [Grafana](#grafana)\n    - [Kibana](#kibana)\n    - [Elasticsearch](#elasticsearch)\n    - [Plugins](#plugins)\n  - [Demo pipeline](#demo-pipeline)\n  - [Plugin customization](#plugin-customization)\n\n## Features\n\n- Visualization of streaming applications, topics, and connectors\n- Monitor all or individual pipelines from multiple namespaces\n- Inspection of Avro schema from schema registry\n- Integration with [streams-bootstrap](https://github.com/bakdata/streams-bootstrap) and [faust-bootstrap](https://github.com/bakdata/faust-bootstrap), or custom streaming app config parsing from Kubernetes deployments using plugins\n- Real-time metrics from Prometheus (consumer lag & read rate, replicas, topic size, messages in & out per second, connector tasks)\n- Linking to external services for logging and analysis, such as Kibana, Grafana, Loki, AKHQ, Redpanda Console, and Elasticsearch\n- Customizable through Python plugins\n\n## Overview\n\nVisit our introduction [blogpost](https://medium.com/bakdata/exploring-data-pipelines-in-apache-kafka-with-streams-explorer-8337dd11fdad) for a complete overview and demo of Streams Explorer.\n\n## Installation\n\n### Docker Compose\n\n1. Forward the ports to Prometheus. (Kafka Connect, Schema Registry, and other integrations are optional)\n2. Start the container\n\n```sh\ndocker compose up\n```\n\nOnce the container is started visit <http://localhost:8080>\n\n### Deploying to Kubernetes cluster\n\n1. Add the Helm chart repository\n\n```sh\nhelm repo add streams-explorer https://bakdata.github.io/streams-explorer\n```\n\n2. Install\n\n```sh\nhelm upgrade --install --values helm-chart/values.yaml streams-explorer streams-explorer/streams-explorer\n```\n\n### Standalone\n\n#### Backend\n\n1. Install dependencies using [Poetry](https://python-poetry.org)\n\n```sh\npoetry install\n```\n\n2. Forward the ports to Prometheus. (Kafka Connect, Schema Registry, and other integrations are optional)\n3. Configure the backend in [settings.yaml](backend/settings.yaml).\n4. Start the backend server\n\n```sh\npoetry run start\n```\n\n#### Frontend\n\n1. Install dependencies\n\n```sh\nnpm ci\n```\n\n2. Start the frontend server\n\n```sh\nnpm run build && npm run prod\n```\n\nVisit <http://localhost:3000>\n\n## Configuration\n\nDepending on your type of installation set the configuration for the backend server in this file:\n\n- **Docker Compose**: [docker-compose.yaml](docker-compose.yaml)\n- **Kubernetes**: [helm-chart/values.yaml](helm-chart/values.yaml)\n- **standalone**: [backend/settings.yaml](backend/settings.yaml)\n\nIn the [helm-chart/values.yaml](helm-chart/values.yaml) configuration is done either through the `config` section using double underscore notation, e.g. `K8S__consumer_group_annotation: consumerGroup` or the content of [backend/settings.yaml](backend/settings.yaml) can be pasted under the `settings` section. Alternatively all configuration options can be written as environment variables using double underscore notation and the prefix `SE`, e.g. `SE_K8S__deployment__cluster=false`.\n\nThe following configuration options are available:\n\n#### General\n\n- `graph.update_interval` Render the graph every x seconds (int, **required**, default: `300`)\n- `graph.layout_arguments` Arguments passed to graphviz layout (string, **required**, default: `-Grankdir=LR -Gnodesep=0.8 -Gpad=10`)\n- `graph.pipeline_distance` Increase/decrease vertical space between pipeline graphs by X pixels (int, **required**, default: `500`)\n- `graph.resolve.input_pattern_topics.all` If true topics that match (extra) input pattern(s) are connected to the streaming app in the graph containing all pipelines (bool, **required**, default: `false`)\n- `graph.resolve.input_pattern_topics.pipelines` If true topics that match (extra) input pattern(s) are connected to the streaming app in pipeline graphs (bool, **required**, default: `false`)\n\n#### Kafka\n\n- `kafka.enable` Enable Kafka (bool, default: `false`)\n- `kafka.config` librdkafka configuration properties ([reference](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md)) (dict, default: `{"bootstrap.servers": "localhost:9092"}`)\n- `kafka.displayed_information` Configuration options of Kafka topics displayed in the frontend (list of dict)\n- `kafka.topic_names_cache.ttl` Cache for retrieving all topic names (used when input topic patterns are resolved) (int, default: `3600`)\n\n#### Kafka Connect\n\n- `kafkaconnect.url` URL of Kafka Connect server (string, default: None)\n- `kafkaconnect.update_interval` Fetch connectors every x seconds (int, default: `300`)\n- `kafkaconnect.displayed_information` Configuration options of Kafka connectors displayed in the frontend (list of dict)\n\n#### Kubernetes\n\n- `k8s.deployment.cluster` Whether streams-explorer is deployed to Kubernetes cluster (bool, **required**, default: `false`)\n- `k8s.deployment.context` Name of cluster (string, optional if running in cluster, default: `kubernetes-cluster`)\n- `k8s.deployment.namespaces` Kubernetes namespaces (list of string, **required**, default: `[\'kubernetes-namespace\']`)\n- `k8s.containers.ignore` Name of containers that should be ignored/hidden (list of string, default: `[\'prometheus-jmx-exporter\']`)\n- `k8s.displayed_information` Details of pod that should be displayed (list of dict, default: `[{\'name\': \'Labels\', \'key\': \'metadata.labels\'}]`)\n- `k8s.labels` Labels used to set attributes of nodes (list of string, **required**, default: `[\'pipeline\']`)\n- `k8s.pipeline.label` Attribute of nodes the pipeline name should be extracted from (string, **required**, default: `pipeline`)\n- `k8s.consumer_group_annotation` Annotation the consumer group name should be extracted from (string, **required**, default: `consumerGroup`)\n\n#### Schema Registry / Karapace\n\n- `schemaregistry.url` URL of Confluent Schema Registry or Karapace (string, default: None)\n\n#### Prometheus\n\n- `prometheus.url` URL of Prometheus (string, **required**, default: `http://localhost:9090`)\n\nThe following exporters are required to collect Kafka metrics for Prometheus:\n\n- [Kafka Exporter](https://github.com/danielqsj/kafka_exporter)\n- [Kafka Lag Exporter](https://github.com/lightbend/kafka-lag-exporter)\n- [Kafka Connect Exporter](https://github.com/wakeful/kafka_connect_exporter)\n\n#### AKHQ\n\n- `akhq.enable` Enable AKHQ (bool, default: `false`)\n- `akhq.url` URL of AKHQ (string, default: `http://localhost:8080`)\n- `akhq.cluster` Name of cluster (string, default: `kubernetes-cluster`)\n- `akhq.connect` Name of connect (string, default: None)\n\n#### Redpanda Console\n\nRedpanda Console can be used instead of AKHQ. (mutually exclusive)\n\n- `redpanda_console.enable` Enable Redpanda Console (bool, default: `false`)\n- `redpanda_console.url` URL of Redpanda Console (string, default: `http://localhost:8080`)\n\n#### Grafana\n\n- `grafana.enable` Enable Grafana (bool, default: `false`)\n- `grafana.url` URL of Grafana (string, default: `http://localhost:3000`)\n- `grafana.dashboards.topics` Path to topics dashboard (string), sample dashboards for topics and consumer groups are included in the [`./grafana`](https://github.com/bakdata/streams-explorer/tree/main/grafana) subfolder\n- `grafana.dashboards.consumergroups` Path to consumer groups dashboard (string)\n\n#### Kibana\n\n- `kibanalogs.enable` Enable Kibana logs (bool, default: `false`)\n- `kibanalogs.url` URL of Kibana logs (string, default: `http://localhost:5601`)\n\n#### Loki\n\nLoki can be used instead of Kibana. (mutually exclusive)\n\n- `loki.enable` Enable Loki logs (bool, default: `false`)\n- `loki.url` URL of Loki logs (string, default: `http://localhost:3000`)\n\n#### Elasticsearch\n\nfor Kafka Connect Elasticsearch connector\n\n- `esindex.url` URL of Elasticsearch index (string, default: `http://localhost:5601/app/kibana#/dev_tools/console`)\n\n#### Plugins\n\n- `plugins.path` Path to folder containing plugins relative to backend (string, **required**, default: `./plugins`)\n- `plugins.extractors.default` Whether to load default extractors (bool, **required**, default: `true`)\n\n## Demo pipeline\n\n![demo-pipeline](https://github.com/bakdata/streams-explorer/blob/main/screens/demo-pipeline.png?raw=true)\n\n[ATM Fraud detection with streams-bootstrap](https://github.com/bakdata/streams-explorer/blob/main/demo-atm-fraud/README.md)\n\n## Plugin customization\n\nIt is possible to create your own config parser, linker, metric provider, and extractors in Python by implementing the `K8sConfigParser`, `LinkingService`, `MetricProvider`, or `Extractor` classes. This way you can customize it to your specific setup and services. As an example we provide the [`DefaultLinker`](https://github.com/bakdata/streams-explorer/blob/main/backend/streams_explorer/defaultlinker.py) as `LinkingService`. The default [`MetricProvider`](https://github.com/bakdata/streams-explorer/blob/main/backend/streams_explorer/core/services/metric_providers.py) supports Prometheus. Furthermore the following default `Extractor` plugins are included:\n\n- [`ElasticsearchSink`](https://github.com/bakdata/streams-explorer/blob/main/backend/streams_explorer/core/extractor/default/elasticsearch_sink.py)\n- [`JdbcSink`](https://github.com/bakdata/streams-explorer/blob/main/backend/streams_explorer/core/extractor/default/jdbc_sink.py)\n- [`S3Sink`](https://github.com/bakdata/streams-explorer/blob/main/backend/streams_explorer/core/extractor/default/s3_sink.py)\n- [`GenericSink`/`GenericSource`](https://github.com/bakdata/streams-explorer/blob/main/backend/streams_explorer/core/extractor/default/generic.py)\n\nIf your streaming application deployments are configured through environment variables, following the schema of [streams-bootstrap](https://github.com/bakdata/streams-bootstrap) or [faust-bootstrap](https://github.com/bakdata/faust-bootstrap), the Streams Explorer works out-of-the-box with the default deployment parser.\nFor streams-bootstrap deployments configured through CLI arguments a separate parser can be loaded by creating a Python file (e.g. `config_parser.py`) in the plugins folder with the following import statement:\n\n```python\nfrom streams_explorer.core.k8s_config_parser import StreamsBootstrapArgsParser\n```\n\nFor other setups a custom config parser plugin can be created by inheriting from the [`K8sConfigParser`](https://github.com/bakdata/streams-explorer/blob/main/backend/streams_explorer/core/k8s_config_parser.py) class and implementing the `parse` method. In this example we\'re retrieving the streaming app configurations from an external REST API. In order for a deployment to be indentified as streaming app, input and output topics are required.\n\n```python\nimport httpx\n\nfrom streams_explorer.core.k8s_config_parser import K8sConfigParser\nfrom streams_explorer.models.k8s import K8sConfig\n\n\nclass CustomConfigParser(K8sConfigParser):\n    def get_name(self) -> str:\n        name = self.k8s_app.metadata.name\n        if not name:\n            raise TypeError(f"Name is required for {self.k8s_app.class_name}")\n        return name\n\n    def parse(self) -> K8sConfig:\n        """Retrieve app config from REST endpoint."""\n        name = self.get_name()\n        data = httpx.get(f"url/config/{name}").json()\n        return K8sConfig(**data)\n```\n',
    'author': 'bakdata',
    'author_email': 'None',
    'maintainer': 'None',
    'maintainer_email': 'None',
    'url': 'https://github.com/bakdata/streams-explorer',
    'packages': packages,
    'package_data': package_data,
    'install_requires': install_requires,
    'entry_points': entry_points,
    'python_requires': '>=3.10,<4.0',
}


setup(**setup_kwargs)
