Flyte Datacatalog Configuration#

application
database
datacatalog
logger
otel
storage

Section: application#

grpcPort (int)#

On which grpc port to serve Catalog

Default Value:

"8081"

grpcServerReflection (bool)#

Enable GRPC Server Reflection

Default Value:

"true"

httpPort (int)#

On which http port to serve Catalog

Default Value:

"8080"

secure (bool)#

Whether to run Catalog in secure mode or not

Default Value:

"false"

readHeaderTimeoutSeconds (int)#

The amount of time allowed to read request headers.

Default Value:

"32"

Section: database#

host (string)#

Default Value:

""

port (int)#

Default Value:

"0"

dbname (string)#

Default Value:

""

username (string)#

Default Value:

""

password (string)#

Default Value:

""

passwordPath (string)#

Default Value:

""

options (string)#

Default Value:

""

debug (bool)#

Default Value:

"false"

enableForeignKeyConstraintWhenMigrating (bool)#

Whether to enable gorm foreign keys when migrating the db

Default Value:

"false"

maxIdleConnections (int)#

maxIdleConnections sets the maximum number of connections in the idle connection pool.

Default Value:

"10"

maxOpenConnections (int)#

maxOpenConnections sets the maximum number of open connections to the database.

Default Value:

"100"

connMaxLifeTime (config.Duration)#

sets the maximum amount of time a connection may be reused

Default Value:

1h0m0s

postgres (database.PostgresConfig)#

Default Value:

dbname: postgres
debug: false
host: localhost
options: sslmode=disable
password: postgres
passwordPath: ""
port: 30001
username: postgres

sqlite (database.SQLiteConfig)#

Default Value:

file: ""

config.Duration#

Duration (int64)#

Default Value:

1h0m0s

database.PostgresConfig#

host (string)#

The host name of the database server

Default Value:

localhost

port (int)#

The port name of the database server

Default Value:

"30001"

dbname (string)#

The database name

Default Value:

postgres

username (string)#

The database user who is connecting to the server.

Default Value:

postgres

password (string)#

The database password.

Default Value:

postgres

passwordPath (string)#

Points to the file containing the database password.

Default Value:

""

options (string)#

See http://gorm.io/docs/connecting_to_the_database.html for available options passed, in addition to the above.

Default Value:

sslmode=disable

debug (bool)#

Whether or not to start the database connection with debug mode enabled.

Default Value:

"false"

database.SQLiteConfig#

file (string)#

The path to the file (existing or new) where the DB should be created / stored. If existing, then this will be re-used, else a new will be created

Default Value:

""

Section: datacatalog#

storage-prefix (string)#

StoragePrefix specifies the prefix where DataCatalog stores offloaded ArtifactData in CloudStorage. If not specified, the data will be stored in the base container directly.

Default Value:

metadata

metrics-scope (string)#

Scope that the metrics will record under.

Default Value:

datacatalog

profiler-port (int)#

Port that the profiling service is listening on.

Default Value:

"10254"

heartbeat-grace-period-multiplier (int)#

Number of heartbeats before a reservation expires without an extension.

Default Value:

"3"

max-reservation-heartbeat (config.Duration)#

The maximum available reservation extension heartbeat interval.

Default Value:

10s

Section: logger#

show-source (bool)#

Includes source code location in logs.

Default Value:

"false"

mute (bool)#

Mutes all logs regardless of severity. Intended for benchmarks/tests only.

Default Value:

"false"

level (int)#

Sets the minimum logging level.

Default Value:

"3"

formatter (logger.FormatterConfig)#

Sets logging format.

Default Value:

type: json

logger.FormatterConfig#

type (string)#

Sets logging format type.

Default Value:

json

Section: otel#

type (string)#

Sets the type of exporter to configure [noop/file/jaeger].

Default Value:

noop

file (otelutils.FileConfig)#

Configuration for exporting telemetry traces to a file

Default Value:

filename: /tmp/trace.txt

jaeger (otelutils.JaegerConfig)#

Configuration for exporting telemetry traces to a jaeger

Default Value:

endpoint: http://localhost:14268/api/traces

otelutils.FileConfig#

filename (string)#

Filename to store exported telemetry traces

Default Value:

/tmp/trace.txt

otelutils.JaegerConfig#

endpoint (string)#

Endpoint for the jaeger telemtry trace ingestor

Default Value:

http://localhost:14268/api/traces

Section: storage#

type (string)#

Sets the type of storage to configure [s3/minio/local/mem/stow].

Default Value:

s3

connection (storage.ConnectionConfig)#

Default Value:

access-key: ""
auth-type: iam
disable-ssl: false
endpoint: ""
region: us-east-1
secret-key: ""

stow (storage.StowConfig)#

Storage config for stow backend.

Default Value:

{}

container (string)#

Initial container (in s3 a bucket) to create -if it doesn’t exist-.’

Default Value:

""

enable-multicontainer (bool)#

If this is true, then the container argument is overlooked and redundant. This config will automatically open new connections to new containers/buckets as they are encountered

Default Value:

"false"

cache (storage.CachingConfig)#

Default Value:

max_size_mbs: 0
target_gc_percent: 0

limits (storage.LimitsConfig)#

Sets limits for stores.

Default Value:

maxDownloadMBs: 2

defaultHttpClient (storage.HTTPClientConfig)#

Sets the default http client config.

Default Value:

headers: null
timeout: 0s

signedUrl (storage.SignedURLConfig)#

Sets config for SignedURL.

Default Value:

{}

storage.CachingConfig#

max_size_mbs (int)#

Maximum size of the cache where the Blob store data is cached in-memory. If not specified or set to 0, cache is not used

Default Value:

"0"

target_gc_percent (int)#

Sets the garbage collection target percentage.

Default Value:

"0"

storage.ConnectionConfig#

endpoint (config.URL)#

URL for storage client to connect to.

Default Value:

""

auth-type (string)#

Auth Type to use [iam,accesskey].

Default Value:

iam

access-key (string)#

Access key to use. Only required when authtype is set to accesskey.

Default Value:

""

secret-key (string)#

Secret to use when accesskey is set.

Default Value:

""

region (string)#

Region to connect to.

Default Value:

us-east-1

disable-ssl (bool)#

Disables SSL connection. Should only be used for development.

Default Value:

"false"

config.URL#

URL (url.URL)#

Default Value:

ForceQuery: false
Fragment: ""
Host: ""
OmitHost: false
Opaque: ""
Path: ""
RawFragment: ""
RawPath: ""
RawQuery: ""
Scheme: ""
User: null

url.URL#

Scheme (string)#

Default Value:

""

Opaque (string)#

Default Value:

""

User (url.Userinfo)#

Default Value:

null

Host (string)#

Default Value:

""

Path (string)#

Default Value:

""

RawPath (string)#

Default Value:

""

OmitHost (bool)#

Default Value:

"false"

ForceQuery (bool)#

Default Value:

"false"

RawQuery (string)#

Default Value:

""

Fragment (string)#

Default Value:

""

RawFragment (string)#

Default Value:

""

storage.HTTPClientConfig#

headers (map[string][]string)#

Default Value:

null

timeout (config.Duration)#

Sets time out on the http client.

Default Value:

0s

storage.LimitsConfig#

maxDownloadMBs (int64)#

Maximum allowed download size (in MBs) per call.

Default Value:

"2"

storage.SignedURLConfig#

stowConfigOverride (map[string]string)#

Default Value:

null

storage.StowConfig#

kind (string)#

Kind of Stow backend to use. Refer to github/flyteorg/stow

Default Value:

""

config (map[string]string)#

Configuration for stow backend. Refer to github/flyteorg/stow

Default Value:

{}