mirror of
https://github.com/twitter/the-algorithm.git
synced 2025-06-10 14:48:16 -05:00
Open-sourcing Unified User Actions
Unified User Action (UUA) is a centralized, real-time stream of user actions on Twitter, consumed by various product, ML, and marketing teams. UUA makes sure all internal teams consume the uniformed user actions data in an accurate and fast way.
This commit is contained in:
twitter-team
@ -0,0 +1,46 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-kill-staging-services",
|
||||
"config-files": [],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 1"
|
||||
},
|
||||
"dependencies": [],
|
||||
"steps": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "script",
|
||||
"name": "uua-kill-staging-services",
|
||||
"keytab": "/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab",
|
||||
"repository": "source",
|
||||
"command": "bash unified_user_actions/scripts/kill_staging.sh",
|
||||
"dependencies": [{
|
||||
"version": "latest",
|
||||
"role": "aurora",
|
||||
"name": "aurora"
|
||||
}],
|
||||
"timeout": "10.minutes"
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "unified_user_actions_dev"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "unified_user_actions_dev"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,66 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "rekey-uua-iesource-prod",
|
||||
"config-files": [
|
||||
"rekey-uua-iesource.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 2"
|
||||
},
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:rekey-uua-iesource"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "rekey-uua-iesource",
|
||||
"artifact": "./dist/rekey-uua-iesource.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "prod",
|
||||
"targets": [
|
||||
{
|
||||
"name": "rekey-uua-iesource-prod-atla",
|
||||
"key": "atla/discode/prod/rekey-uua-iesource"
|
||||
},
|
||||
{
|
||||
"name": "rekey-uua-iesource-prod-pdxa",
|
||||
"key": "pdxa/discode/prod/rekey-uua-iesource"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "discode-oncall"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "discode-oncall"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "rekey-uua-iesource-staging",
|
||||
"config-files": [
|
||||
"rekey-uua-iesource.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:rekey-uua-iesource"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "rekey-uua-iesource-staging",
|
||||
"artifact": "./dist/rekey-uua-iesource.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "rekey-uua-iesource-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/rekey-uua-iesource"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
204
unified_user_actions/service/deploy/rekey-uua-iesource.aurora
Normal file
204
unified_user_actions/service/deploy/rekey-uua-iesource.aurora
Normal file
@ -0,0 +1,204 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'rekey-uua-iesource'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk.
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 250)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/cdm-1:kafka-tls')
|
||||
kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
source_topic = Default(String, 'interaction_events')
|
||||
sink_topics = Default(String, 'uua_keyed')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = DISK_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}'
|
||||
' -kafka.group.id={{name}}-{{environment}}-{{cluster}}'
|
||||
' -kafka.producer.client.id={{name}}-{{environment}}'
|
||||
' -kafka.max.pending.requests=10000'
|
||||
' -kafka.consumer.fetch.max=1.megabytes'
|
||||
' -kafka.producer.batch.size=16.kilobytes'
|
||||
' -kafka.producer.buffer.mem=128.megabytes'
|
||||
' -kafka.producer.linger=50.milliseconds'
|
||||
' -kafka.producer.request.timeout=30.seconds'
|
||||
' -kafka.producer.compression.type=lz4'
|
||||
' -kafka.worker.threads=5'
|
||||
' -kafka.source.topic={{profile.source_topic}}'
|
||||
' -kafka.sink.topics={{profile.sink_topics}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' -cluster={{cluster}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 500,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers_remote_dest = '/srv#/devel/local/kafka/ingestion-1:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'DEBUG',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
|
||||
### pdxa right now doesn't have InteractionEvents topic
|
||||
PRODUCTION_PDXA = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml',
|
||||
kafka_bootstrap_servers = '/srv#/prod/atla/kafka/cdm-1:kafka-tls'
|
||||
)
|
||||
|
||||
STAGING_PDXA = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers = '/srv#/prod/atla/kafka/cdm-1:kafka-tls',
|
||||
kafka_bootstrap_servers_remote_dest = '/srv#/devel/local/kafka/ingestion-1:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL_PDXA = STAGING(
|
||||
log_level = 'DEBUG',
|
||||
kafka_bootstrap_servers = '/srv#/prod/atla/kafka/cdm-1:kafka-tls'
|
||||
)
|
||||
|
||||
prod_job_pdxa = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION_PDXA)
|
||||
|
||||
staging_job_pdxa = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING_PDXA)
|
||||
|
||||
devel_job_pdxa = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL_PDXA)
|
||||
|
||||
jobs.append(prod_job_pdxa(cluster = 'pdxa'))
|
||||
jobs.append(staging_job_pdxa(cluster = 'pdxa'))
|
||||
jobs.append(devel_job_pdxa(cluster = 'pdxa'))
|
||||
66
unified_user_actions/service/deploy/rekey-uua-prod.workflow
Normal file
66
unified_user_actions/service/deploy/rekey-uua-prod.workflow
Normal file
@ -0,0 +1,66 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "rekey-uua-prod",
|
||||
"config-files": [
|
||||
"rekey-uua.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 2"
|
||||
},
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:rekey-uua"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "rekey-uua",
|
||||
"artifact": "./dist/rekey-uua.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "prod",
|
||||
"targets": [
|
||||
{
|
||||
"name": "rekey-uua-prod-atla",
|
||||
"key": "atla/discode/prod/rekey-uua"
|
||||
},
|
||||
{
|
||||
"name": "rekey-uua-prod-pdxa",
|
||||
"key": "pdxa/discode/prod/rekey-uua"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "discode-oncall"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "discode-oncall"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "rekey-uua-staging",
|
||||
"config-files": [
|
||||
"rekey-uua.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:rekey-uua"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "rekey-uua-staging",
|
||||
"artifact": "./dist/rekey-uua.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "rekey-uua-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/rekey-uua"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
167
unified_user_actions/service/deploy/rekey-uua.aurora
Normal file
167
unified_user_actions/service/deploy/rekey-uua.aurora
Normal file
@ -0,0 +1,167 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'rekey-uua'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk.
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 100)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
source_topic = Default(String, 'unified_user_actions')
|
||||
sink_topics = Default(String, 'uua_keyed')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = DISK_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}'
|
||||
' -kafka.group.id={{name}}-{{environment}}'
|
||||
' -kafka.producer.client.id={{name}}-{{environment}}'
|
||||
' -kafka.max.pending.requests=10000'
|
||||
' -kafka.consumer.fetch.max=1.megabytes'
|
||||
' -kafka.producer.batch.size=16.kilobytes'
|
||||
' -kafka.producer.buffer.mem=128.megabytes'
|
||||
' -kafka.producer.linger=50.milliseconds'
|
||||
' -kafka.producer.request.timeout=30.seconds'
|
||||
' -kafka.producer.compression.type=lz4'
|
||||
' -kafka.worker.threads=5'
|
||||
' -kafka.source.topic={{profile.source_topic}}'
|
||||
' -kafka.sink.topics={{profile.sink_topics}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' -cluster={{cluster}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 100,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers_remote_dest = '/srv#/devel/local/kafka/ingestion-1:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'DEBUG',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
@ -0,0 +1,66 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-ads-callback-engagements-prod",
|
||||
"config-files": [
|
||||
"uua-ads-callback-engagements.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 2"
|
||||
},
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-ads-callback-engagements"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-ads-callback-engagements",
|
||||
"artifact": "./dist/uua-ads-callback-engagements.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "prod",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-ads-callback-engagements-prod-atla",
|
||||
"key": "atla/discode/prod/uua-ads-callback-engagements"
|
||||
},
|
||||
{
|
||||
"name": "uua-ads-callback-engagements-prod-pdxa",
|
||||
"key": "pdxa/discode/prod/uua-ads-callback-engagements"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "discode-oncall"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "discode-oncall"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-ads-callback-engagements-staging",
|
||||
"config-files": [
|
||||
"uua-ads-callback-engagements.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-ads-callback-engagements"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-ads-callback-engagements-staging",
|
||||
"artifact": "./dist/uua-ads-callback-engagements.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-ads-callback-engagements-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/uua-ads-callback-engagements"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,167 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'uua-ads-callback-engagements'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk.
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 50)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/ads-callback-1:kafka-tls')
|
||||
kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
source_topic = Default(String, 'ads_spend_prod')
|
||||
sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = DISK_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}'
|
||||
' -kafka.group.id={{name}}-{{environment}}'
|
||||
' -kafka.producer.client.id={{name}}-{{environment}}'
|
||||
' -kafka.max.pending.requests=10000'
|
||||
' -kafka.consumer.fetch.max=1.megabytes'
|
||||
' -kafka.producer.batch.size=16.kilobytes'
|
||||
' -kafka.producer.buffer.mem=128.megabytes'
|
||||
' -kafka.producer.linger=50.milliseconds'
|
||||
' -kafka.producer.request.timeout=30.seconds'
|
||||
' -kafka.producer.compression.type=lz4'
|
||||
' -kafka.worker.threads=5'
|
||||
' -kafka.source.topic={{profile.source_topic}}'
|
||||
' -kafka.sink.topics={{profile.sink_topics}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' -cluster={{cluster}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 50,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'DEBUG',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
@ -0,0 +1,66 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-client-event-prod",
|
||||
"config-files": [
|
||||
"uua-client-event.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 2"
|
||||
},
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-client-event"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-client-event",
|
||||
"artifact": "./dist/uua-client-event.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "prod",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-client-event-prod-atla",
|
||||
"key": "atla/discode/prod/uua-client-event"
|
||||
},
|
||||
{
|
||||
"name": "uua-client-event-prod-pdxa",
|
||||
"key": "pdxa/discode/prod/uua-client-event"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "discode-oncall"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "discode-oncall"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-client-event-staging",
|
||||
"config-files": [
|
||||
"uua-client-event.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-client-event"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-client-event-staging",
|
||||
"artifact": "./dist/uua-client-event.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-client-event-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/uua-client-event"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
174
unified_user_actions/service/deploy/uua-client-event.aurora
Normal file
174
unified_user_actions/service/deploy/uua-client-event.aurora
Normal file
@ -0,0 +1,174 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'uua-client-event'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk.
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 1000)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/client-events:kafka-tls')
|
||||
kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
source_topic = Default(String, 'client_event')
|
||||
sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = DISK_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:MaxMetaspaceSize=536870912'
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}'
|
||||
' -kafka.group.id={{name}}-{{environment}}'
|
||||
' -kafka.producer.client.id={{name}}-{{environment}}'
|
||||
' -kafka.max.pending.requests=10000'
|
||||
# CE events is about 0.4-0.6kb per message on the consumer side. A fetch size of 6~18 MB get us
|
||||
# about 10k ~ 20k of messages per batch. This fits the size of our pending requests queue and
|
||||
# within the limit of the max poll records.
|
||||
' -kafka.consumer.fetch.max=9.megabytes'
|
||||
' -kafka.consumer.fetch.min=3.megabytes'
|
||||
' -kafka.max.poll.records=40000'
|
||||
' -kafka.commit.interval=20.seconds'
|
||||
' -kafka.producer.batch.size=4.megabytes'
|
||||
' -kafka.producer.buffer.mem=64.megabytes'
|
||||
' -kafka.producer.linger=100.millisecond'
|
||||
' -kafka.producer.request.timeout=30.seconds'
|
||||
' -kafka.producer.compression.type=lz4'
|
||||
' -kafka.worker.threads=4'
|
||||
' -kafka.source.topic={{profile.source_topic}}'
|
||||
' -kafka.sink.topics={{profile.sink_topics}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' -cluster={{cluster}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 1000,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'INFO',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
@ -0,0 +1,66 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-email-notification-event-prod",
|
||||
"config-files": [
|
||||
"uua-email-notification-event.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 2"
|
||||
},
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-email-notification-event"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-email-notification-event",
|
||||
"artifact": "./dist/uua-email-notification-event.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "prod",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-email-notification-event-prod-atla",
|
||||
"key": "atla/discode/prod/uua-email-notification-event"
|
||||
},
|
||||
{
|
||||
"name": "uua-email-notification-event-prod-pdxa",
|
||||
"key": "pdxa/discode/prod/uua-email-notification-event"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "discode-oncall"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "discode-oncall"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-email-notification-event-staging",
|
||||
"config-files": [
|
||||
"uua-email-notification-event.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-email-notification-event"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-email-notification-event-staging",
|
||||
"artifact": "./dist/uua-email-notification-event.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-email-notification-event-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/uua-email-notification-event"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,169 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'uua-email-notification-event'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk.
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 20)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/main-2:kafka-tls')
|
||||
kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
source_topic = Default(String, 'notifications')
|
||||
sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = RAM_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}'
|
||||
' -kafka.group.id={{name}}-{{environment}}'
|
||||
' -kafka.producer.client.id={{name}}-{{environment}}'
|
||||
' -kafka.max.pending.requests=10000'
|
||||
' -kafka.consumer.fetch.max=1.megabytes'
|
||||
' -kafka.max.poll.records=20000'
|
||||
' -kafka.commit.interval=10.seconds'
|
||||
' -kafka.producer.batch.size=16.kilobytes'
|
||||
' -kafka.producer.buffer.mem=64.megabytes'
|
||||
' -kafka.producer.linger=0.milliseconds'
|
||||
' -kafka.producer.request.timeout=30.seconds'
|
||||
' -kafka.producer.compression.type=lz4'
|
||||
' -kafka.worker.threads=5'
|
||||
' -kafka.source.topic={{profile.source_topic}}'
|
||||
' -kafka.sink.topics={{profile.sink_topics}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' -cluster={{cluster}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 50,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'INFO',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-enricher-staging",
|
||||
"config-files": [
|
||||
"uua-enricher.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-enricher"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-enricher-staging",
|
||||
"artifact": "./dist/uua-enricher.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-enricher-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/uua-enricher"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
151
unified_user_actions/service/deploy/uua-enricher.aurora
Normal file
151
unified_user_actions/service/deploy/uua-enricher.aurora
Normal file
@ -0,0 +1,151 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'uua-enricher'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 6 * GB
|
||||
RAM_SIZE = 8 * GB
|
||||
DISK_SIZE = 3 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 10)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = DISK_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.application.id={{name}}.{{environment}}'
|
||||
' -kafka.application.num.instances={{instances}}' # Used for static partitioning
|
||||
' -kafka.application.server={{mesos.instance}}.{{name}}.{{environment}}.{{role}}.service.{{cluster}}.twitter.com:80'
|
||||
' -com.twitter.finatra.kafkastreams.config.principal={{role}}'
|
||||
' -thrift.client.id={{name}}.{{environment}}'
|
||||
' -service.identifier="{{role}}:{{name}}:{{environment}}:{{cluster}}"'
|
||||
' -local.cache.ttl.seconds=86400'
|
||||
' -local.cache.max.size=400000000'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 50,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers = '/s/kafka/custdevel:kafka-tls'
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'DEBUG',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-enrichment-planner-staging",
|
||||
"config-files": [
|
||||
"uua-enrichment-planner.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-enrichment-planner"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-enrichment-planner-staging",
|
||||
"artifact": "./dist/uua-enrichment-planner.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-enricher-enrichment-planner-pdxa",
|
||||
"key": "pdxa/discode/staging/uua-enrichment-planner"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,156 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'uua-enrichment-planner'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 50)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
kafka_output_server = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = DISK_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version(default_version='live')
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem'
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.output.server={{profile.kafka_output_server}}'
|
||||
' -kafka.application.id=uua-enrichment-planner'
|
||||
' -com.twitter.finatra.kafkastreams.config.principal={{role}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 50,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_output_server = '/s/kafka/custdevel:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'DEBUG',
|
||||
instances = 2,
|
||||
kafka_output_server = '/s/kafka/custdevel:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
@ -0,0 +1,66 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-favorite-archival-events-prod",
|
||||
"config-files": [
|
||||
"uua-favorite-archival-events.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 2"
|
||||
},
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-favorite-archival-events"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-favorite-archival-events",
|
||||
"artifact": "./dist/uua-favorite-archival-events.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "prod",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-favorite-archival-events-prod-atla",
|
||||
"key": "atla/discode/prod/uua-favorite-archival-events"
|
||||
},
|
||||
{
|
||||
"name": "uua-favorite-archival-events-prod-pdxa",
|
||||
"key": "pdxa/discode/prod/uua-favorite-archival-events"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "discode-oncall"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "discode-oncall"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-favorite-archival-events-staging",
|
||||
"config-files": [
|
||||
"uua-favorite-archival-events.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-favorite-archival-events"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-favorite-archival-events-staging",
|
||||
"artifact": "./dist/uua-favorite-archival-events.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-favorite-archival-events-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/uua-favorite-archival-events"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,167 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'uua-favorite-archival-events'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk.
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 10)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/main-2:kafka-tls')
|
||||
kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
source_topic = Default(String, 'favorite_archival_events')
|
||||
sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = RAM_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}'
|
||||
' -kafka.group.id={{name}}-{{environment}}'
|
||||
' -kafka.producer.client.id={{name}}-{{environment}}'
|
||||
' -kafka.max.pending.requests=10000'
|
||||
' -kafka.consumer.fetch.max=1.megabytes'
|
||||
' -kafka.producer.batch.size=16.kilobytes'
|
||||
' -kafka.producer.buffer.mem=128.megabytes'
|
||||
' -kafka.producer.linger=0.milliseconds'
|
||||
' -kafka.producer.request.timeout=30.seconds'
|
||||
' -kafka.producer.compression.type=lz4'
|
||||
' -kafka.worker.threads=5'
|
||||
' -kafka.source.topic={{profile.source_topic}}'
|
||||
' -kafka.sink.topics={{profile.sink_topics}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' -cluster={{cluster}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 50,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'INFO',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
@ -0,0 +1,66 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-retweet-archival-events-prod",
|
||||
"config-files": [
|
||||
"uua-retweet-archival-events.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 2"
|
||||
},
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-retweet-archival-events"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-retweet-archival-events",
|
||||
"artifact": "./dist/uua-retweet-archival-events.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "prod",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-retweet-archival-events-prod-atla",
|
||||
"key": "atla/discode/prod/uua-retweet-archival-events"
|
||||
},
|
||||
{
|
||||
"name": "uua-retweet-archival-events-prod-pdxa",
|
||||
"key": "pdxa/discode/prod/uua-retweet-archival-events"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "discode-oncall"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "discode-oncall"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-retweet-archival-events-staging",
|
||||
"config-files": [
|
||||
"uua-retweet-archival-events.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-retweet-archival-events"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-retweet-archival-events-staging",
|
||||
"artifact": "./dist/uua-retweet-archival-events.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-retweet-archival-events-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/uua-retweet-archival-events"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,167 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'uua-retweet-archival-events'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk.
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 10)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/main-2:kafka-tls')
|
||||
kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
source_topic = Default(String, 'retweet_archival_events')
|
||||
sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = RAM_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}'
|
||||
' -kafka.group.id={{name}}-{{environment}}'
|
||||
' -kafka.producer.client.id={{name}}-{{environment}}'
|
||||
' -kafka.max.pending.requests=10000'
|
||||
' -kafka.consumer.fetch.max=1.megabytes'
|
||||
' -kafka.producer.batch.size=16.kilobytes'
|
||||
' -kafka.producer.buffer.mem=128.megabytes'
|
||||
' -kafka.producer.linger=0.milliseconds'
|
||||
' -kafka.producer.request.timeout=30.seconds'
|
||||
' -kafka.producer.compression.type=lz4'
|
||||
' -kafka.worker.threads=5'
|
||||
' -kafka.source.topic={{profile.source_topic}}'
|
||||
' -kafka.sink.topics={{profile.sink_topics}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' -cluster={{cluster}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 50,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'INFO',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
@ -0,0 +1,66 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-social-graph-prod",
|
||||
"config-files": [
|
||||
"uua-social-graph.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 2"
|
||||
},
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-social-graph"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-social-graph",
|
||||
"artifact": "./dist/uua-social-graph.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "prod",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-social-graph-prod-atla",
|
||||
"key": "atla/discode/prod/uua-social-graph"
|
||||
},
|
||||
{
|
||||
"name": "uua-social-graph-prod-pdxa",
|
||||
"key": "pdxa/discode/prod/uua-social-graph"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "discode-oncall"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "discode-oncall"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-social-graph-staging",
|
||||
"config-files": [
|
||||
"uua-social-graph.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-social-graph"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-social-graph-staging",
|
||||
"artifact": "./dist/uua-social-graph.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-social-graph-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/uua-social-graph"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
167
unified_user_actions/service/deploy/uua-social-graph.aurora
Normal file
167
unified_user_actions/service/deploy/uua-social-graph.aurora
Normal file
@ -0,0 +1,167 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'uua-social-graph'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk.
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 20)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
source_topic = Default(String, 'social_write_event')
|
||||
sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = RAM_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}'
|
||||
' -kafka.group.id={{name}}-{{environment}}'
|
||||
' -kafka.producer.client.id={{name}}-{{environment}}'
|
||||
' -kafka.max.pending.requests=10000'
|
||||
' -kafka.consumer.fetch.max=1.megabytes'
|
||||
' -kafka.producer.batch.size=16.kilobytes'
|
||||
' -kafka.producer.buffer.mem=128.megabytes'
|
||||
' -kafka.producer.linger=0.second'
|
||||
' -kafka.producer.request.timeout=30.seconds'
|
||||
' -kafka.producer.compression.type=lz4'
|
||||
' -kafka.worker.threads=5'
|
||||
' -kafka.source.topic={{profile.source_topic}}'
|
||||
' -kafka.sink.topics={{profile.sink_topics}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' -cluster={{cluster}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 50,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'INFO',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
@ -0,0 +1,66 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-tls-favs-prod",
|
||||
"config-files": [
|
||||
"uua-tls-favs.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 2"
|
||||
},
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-tls-favs"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-tls-favs",
|
||||
"artifact": "./dist/uua-tls-favs.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "prod",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-tls-favs-prod-atla",
|
||||
"key": "atla/discode/prod/uua-tls-favs"
|
||||
},
|
||||
{
|
||||
"name": "uua-tls-favs-prod-pdxa",
|
||||
"key": "pdxa/discode/prod/uua-tls-favs"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "discode-oncall"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "discode-oncall"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-tls-favs-staging",
|
||||
"config-files": [
|
||||
"uua-tls-favs.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-tls-favs"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-tls-favs-staging",
|
||||
"artifact": "./dist/uua-tls-favs.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-tls-favs-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/uua-tls-favs"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
167
unified_user_actions/service/deploy/uua-tls-favs.aurora
Normal file
167
unified_user_actions/service/deploy/uua-tls-favs.aurora
Normal file
@ -0,0 +1,167 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'uua-tls-favs'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk.
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 20)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/main-1:kafka-tls')
|
||||
kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
source_topic = Default(String, 'timeline_service_favorites')
|
||||
sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = RAM_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}'
|
||||
' -kafka.group.id={{name}}-{{environment}}'
|
||||
' -kafka.producer.client.id={{name}}-{{environment}}'
|
||||
' -kafka.max.pending.requests=10000'
|
||||
' -kafka.consumer.fetch.max=1.megabytes'
|
||||
' -kafka.producer.batch.size=16.kilobytes'
|
||||
' -kafka.producer.buffer.mem=128.megabytes'
|
||||
' -kafka.producer.linger=50.milliseconds'
|
||||
' -kafka.producer.request.timeout=30.seconds'
|
||||
' -kafka.producer.compression.type=lz4'
|
||||
' -kafka.worker.threads=5'
|
||||
' -kafka.source.topic={{profile.source_topic}}'
|
||||
' -kafka.sink.topics={{profile.sink_topics}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' -cluster={{cluster}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 50,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'INFO',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
@ -0,0 +1,66 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-tweetypie-event-prod",
|
||||
"config-files": [
|
||||
"uua-tweetypie-event.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 2"
|
||||
},
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-tweetypie-event"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-tweetypie-event",
|
||||
"artifact": "./dist/uua-tweetypie-event.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "prod",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-tweetypie-event-prod-atla",
|
||||
"key": "atla/discode/prod/uua-tweetypie-event"
|
||||
},
|
||||
{
|
||||
"name": "uua-tweetypie-event-prod-pdxa",
|
||||
"key": "pdxa/discode/prod/uua-tweetypie-event"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "discode-oncall"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "discode-oncall"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-tweetypie-event-staging",
|
||||
"config-files": [
|
||||
"uua-tweetypie-event.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-tweetypie-event"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-tweetypie-event-staging",
|
||||
"artifact": "./dist/uua-tweetypie-event.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-tweetypie-event-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/uua-tweetypie-event"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
167
unified_user_actions/service/deploy/uua-tweetypie-event.aurora
Normal file
167
unified_user_actions/service/deploy/uua-tweetypie-event.aurora
Normal file
@ -0,0 +1,167 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'uua-tweetypie-event'
|
||||
|
||||
CPU_NUM = 2
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk.
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 20)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/tweet-events:kafka-tls')
|
||||
kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
source_topic = Default(String, 'tweet_events')
|
||||
sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = DISK_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}'
|
||||
' -kafka.group.id={{name}}-{{environment}}'
|
||||
' -kafka.producer.client.id={{name}}-{{environment}}'
|
||||
' -kafka.max.pending.requests=10000'
|
||||
' -kafka.consumer.fetch.max=1.megabytes'
|
||||
' -kafka.producer.batch.size=16.kilobytes'
|
||||
' -kafka.producer.buffer.mem=64.megabytes'
|
||||
' -kafka.producer.linger=0.milliseconds'
|
||||
' -kafka.producer.request.timeout=30.seconds'
|
||||
' -kafka.producer.compression.type=lz4'
|
||||
' -kafka.worker.threads=5'
|
||||
' -kafka.source.topic={{profile.source_topic}}'
|
||||
' -kafka.sink.topics={{profile.sink_topics}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' -cluster={{cluster}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 50,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'INFO',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
@ -0,0 +1,66 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-user-modification-prod",
|
||||
"config-files": [
|
||||
"uua-user-modification.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"trigger": {
|
||||
"cron-schedule": "0 17 * * 2"
|
||||
},
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-user-modification"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-user-modification",
|
||||
"artifact": "./dist/uua-user-modification.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "prod",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-user-modification-prod-atla",
|
||||
"key": "atla/discode/prod/uua-user-modification"
|
||||
},
|
||||
{
|
||||
"name": "uua-user-modification-prod-pdxa",
|
||||
"key": "pdxa/discode/prod/uua-user-modification"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"subscriptions": [
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [
|
||||
{
|
||||
"to": "discode-oncall"
|
||||
}
|
||||
],
|
||||
"events": ["WORKFLOW_SUCCESS"]
|
||||
},
|
||||
{
|
||||
"type": "SLACK",
|
||||
"recipients": [{
|
||||
"to": "discode-oncall"
|
||||
}],
|
||||
"events": ["*FAILED"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
{
|
||||
"role": "discode",
|
||||
"name": "uua-user-modification-staging",
|
||||
"config-files": [
|
||||
"uua-user-modification.aurora"
|
||||
],
|
||||
"build": {
|
||||
"play": true,
|
||||
"dependencies": [
|
||||
{
|
||||
"role": "packer",
|
||||
"name": "packer-client-no-pex",
|
||||
"version": "latest"
|
||||
}
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"type": "bazel-bundle",
|
||||
"name": "bundle",
|
||||
"target": "unified_user_actions/service/src/main/scala:uua-user-modification"
|
||||
},
|
||||
{
|
||||
"type": "packer",
|
||||
"name": "uua-user-modification-staging",
|
||||
"artifact": "./dist/uua-user-modification.zip"
|
||||
}
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"type": "group",
|
||||
"name": "staging",
|
||||
"targets": [
|
||||
{
|
||||
"name": "uua-user-modification-staging-pdxa",
|
||||
"key": "pdxa/discode/staging/uua-user-modification"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
167
unified_user_actions/service/deploy/uua-user-modification.aurora
Normal file
167
unified_user_actions/service/deploy/uua-user-modification.aurora
Normal file
@ -0,0 +1,167 @@
|
||||
import os
|
||||
import itertools
|
||||
import subprocess
|
||||
import math
|
||||
|
||||
SERVICE_NAME = 'uua-user-modification'
|
||||
|
||||
CPU_NUM = 3
|
||||
HEAP_SIZE = 3 * GB
|
||||
RAM_SIZE = HEAP_SIZE + 1 * GB
|
||||
# We make disk size larger than HEAP so that if we ever need to do a heap dump, it will fit on disk.
|
||||
DISK_SIZE = HEAP_SIZE + 2 * GB
|
||||
|
||||
class Profile(Struct):
|
||||
package = Default(String, SERVICE_NAME)
|
||||
cmdline_flags = Default(String, '')
|
||||
log_level = Default(String, 'INFO')
|
||||
instances = Default(Integer, 10)
|
||||
kafka_bootstrap_servers = Default(String, '/s/kafka/main-1:kafka-tls')
|
||||
kafka_bootstrap_servers_remote_dest = Default(String, '/s/kafka/bluebird-1:kafka-tls')
|
||||
source_topic = Default(String, 'user_modifications')
|
||||
sink_topics = Default(String, 'unified_user_actions,unified_user_actions_engagements')
|
||||
decider_overlay = Default(String, '')
|
||||
|
||||
resources = Resources(
|
||||
cpu = CPU_NUM,
|
||||
ram = RAM_SIZE,
|
||||
disk = DISK_SIZE
|
||||
)
|
||||
|
||||
install = Packer.install(
|
||||
name = '{{profile.package}}',
|
||||
version = Workflows.package_version()
|
||||
)
|
||||
|
||||
async_profiler_install = Packer.install(
|
||||
name = 'async-profiler',
|
||||
role = 'csl-perf',
|
||||
version = 'latest'
|
||||
)
|
||||
|
||||
setup_jaas_config = Process(
|
||||
name = 'setup_jaas_config',
|
||||
cmdline = '''
|
||||
mkdir -p jaas_config
|
||||
echo "KafkaClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
principal=\\"discode@TWITTER.BIZ\\"
|
||||
useKeyTab=true
|
||||
storeKey=true
|
||||
keyTab=\\"/var/lib/tss/keys/fluffy/keytabs/client/discode.keytab\\"
|
||||
doNotPrompt=true;
|
||||
};" >> jaas_config/jaas.conf
|
||||
'''
|
||||
)
|
||||
|
||||
main = JVMProcess(
|
||||
name = SERVICE_NAME,
|
||||
jvm = Java11(
|
||||
heap = HEAP_SIZE,
|
||||
extra_jvm_flags =
|
||||
'-Djava.net.preferIPv4Stack=true'
|
||||
|
||||
' -XX:+UseNUMA'
|
||||
' -XX:+AggressiveOpts'
|
||||
' -XX:+PerfDisableSharedMem' # http://www.evanjones.ca/jvm-mmap-pause.html
|
||||
|
||||
' -Dlog_level={{profile.log_level}}'
|
||||
' -Dlog.access.output=access.log'
|
||||
' -Dlog.service.output={{name}}.log'
|
||||
' -Djava.security.auth.login.config=jaas_config/jaas.conf'
|
||||
),
|
||||
arguments =
|
||||
'-jar {{name}}-bin.jar'
|
||||
' -admin.port=:{{thermos.ports[health]}}'
|
||||
' -kafka.bootstrap.servers={{profile.kafka_bootstrap_servers}}'
|
||||
' -kafka.bootstrap.servers.remote.dest={{profile.kafka_bootstrap_servers_remote_dest}}'
|
||||
' -kafka.group.id={{name}}-{{environment}}'
|
||||
' -kafka.producer.client.id={{name}}-{{environment}}'
|
||||
' -kafka.max.pending.requests=10000'
|
||||
' -kafka.consumer.fetch.max=1.megabytes'
|
||||
' -kafka.producer.batch.size=16.kilobytes'
|
||||
' -kafka.producer.buffer.mem=128.megabytes'
|
||||
' -kafka.producer.linger=50.milliseconds'
|
||||
' -kafka.producer.request.timeout=30.seconds'
|
||||
' -kafka.producer.compression.type=lz4'
|
||||
' -kafka.worker.threads=5'
|
||||
' -kafka.source.topic={{profile.source_topic}}'
|
||||
' -kafka.sink.topics={{profile.sink_topics}}'
|
||||
' -decider.base=decider.yml'
|
||||
' -decider.overlay={{profile.decider_overlay}}'
|
||||
' -cluster={{cluster}}'
|
||||
' {{profile.cmdline_flags}}',
|
||||
resources = resources
|
||||
)
|
||||
|
||||
stats = Stats(
|
||||
library = 'metrics',
|
||||
port = 'admin'
|
||||
)
|
||||
|
||||
job_template = Service(
|
||||
name = SERVICE_NAME,
|
||||
role = 'discode',
|
||||
instances = '{{profile.instances}}',
|
||||
contact = 'disco-data-eng@twitter.com',
|
||||
constraints = {'rack': 'limit:1', 'host': 'limit:1'},
|
||||
announce = Announcer(
|
||||
primary_port = 'health',
|
||||
portmap = {'aurora': 'health', 'admin': 'health'}
|
||||
),
|
||||
task = Task(
|
||||
resources = resources,
|
||||
name = SERVICE_NAME,
|
||||
processes = [async_profiler_install, install, setup_jaas_config, main, stats],
|
||||
constraints = order(async_profiler_install, install, setup_jaas_config, main)
|
||||
),
|
||||
health_check_config = HealthCheckConfig(
|
||||
initial_interval_secs = 100,
|
||||
interval_secs = 60,
|
||||
timeout_secs = 60,
|
||||
max_consecutive_failures = 4
|
||||
),
|
||||
update_config = UpdateConfig(
|
||||
batch_size = 50,
|
||||
watch_secs = 90,
|
||||
max_per_shard_failures = 3,
|
||||
max_total_failures = 0,
|
||||
rollback_on_failure = False
|
||||
)
|
||||
)
|
||||
|
||||
PRODUCTION = Profile(
|
||||
# go/uua-decider
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/prod/{{cluster}}/decider_overlay.yml'
|
||||
)
|
||||
|
||||
STAGING = Profile(
|
||||
package = SERVICE_NAME+'-staging',
|
||||
cmdline_flags = '',
|
||||
kafka_bootstrap_servers_remote_dest = '/s/kafka/custdevel:kafka-tls',
|
||||
decider_overlay = '/usr/local/config/overlays/discode-default/UnifiedUserActions/staging/{{cluster}}/decider_overlay.yml' # go/uua-decider
|
||||
)
|
||||
|
||||
DEVEL = STAGING(
|
||||
log_level = 'DEBUG',
|
||||
)
|
||||
|
||||
|
||||
prod_job = job_template(
|
||||
tier = 'preferred',
|
||||
environment = 'prod',
|
||||
).bind(profile = PRODUCTION)
|
||||
|
||||
staging_job = job_template(
|
||||
environment = 'staging'
|
||||
).bind(profile = STAGING)
|
||||
|
||||
devel_job = job_template(
|
||||
environment = 'devel'
|
||||
).bind(profile = DEVEL)
|
||||
|
||||
jobs = []
|
||||
for cluster in ['atla', 'pdxa']:
|
||||
jobs.append(prod_job(cluster = cluster))
|
||||
jobs.append(staging_job(cluster = cluster))
|
||||
jobs.append(devel_job(cluster = cluster))
|
||||
13
unified_user_actions/service/src/main/resources/BUILD
Normal file
13
unified_user_actions/service/src/main/resources/BUILD
Normal file
@ -0,0 +1,13 @@
|
||||
resources(
|
||||
sources = ["*.*"],
|
||||
tags = ["bazel-compatible"],
|
||||
)
|
||||
|
||||
files(
|
||||
name = "files",
|
||||
sources = [
|
||||
"!BUILD",
|
||||
"**/*",
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
)
|
||||
324
unified_user_actions/service/src/main/resources/decider.yml
Normal file
324
unified_user_actions/service/src/main/resources/decider.yml
Normal file
@ -0,0 +1,324 @@
|
||||
# Naming convention:
|
||||
# For publishing action types, use [Publish][ActionTypeInThrift]. Please see the Thrift definition at unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions/action_info.thrift
|
||||
|
||||
PublishServerTweetFav:
|
||||
default_availability: 0
|
||||
PublishServerTweetUnfav:
|
||||
default_availability: 0
|
||||
PublishServerTweetCreate:
|
||||
default_availability: 0
|
||||
PublishServerTweetReply:
|
||||
default_availability: 0
|
||||
PublishServerTweetQuote:
|
||||
default_availability: 0
|
||||
PublishServerTweetRetweet:
|
||||
default_availability: 0
|
||||
PublishServerTweetDelete:
|
||||
default_availability: 0
|
||||
PublishServerTweetUnreply:
|
||||
default_availability: 0
|
||||
PublishServerTweetUnquote:
|
||||
default_availability: 0
|
||||
PublishServerTweetUnretweet:
|
||||
default_availability: 0
|
||||
PublishServerTweetEdit:
|
||||
default_availability: 0
|
||||
PublishServerTweetReport:
|
||||
default_availability: 0
|
||||
PublishServerProfileFollow:
|
||||
default_availability: 0
|
||||
PublishServerProfileUnfollow:
|
||||
default_availability: 0
|
||||
PublishServerProfileBlock:
|
||||
default_availability: 0
|
||||
PublishServerProfileUnblock:
|
||||
default_availability: 0
|
||||
PublishServerProfileMute:
|
||||
default_availability: 0
|
||||
PublishServerProfileUnmute:
|
||||
default_availability: 0
|
||||
PublishServerProfileReport:
|
||||
default_availability: 0
|
||||
PublishClientTweetFav:
|
||||
default_availability: 0
|
||||
PublishClientTweetUnfav:
|
||||
default_availability: 0
|
||||
PublishClientTweetLingerImpression:
|
||||
default_availability: 0
|
||||
PublishClientTweetRenderImpression:
|
||||
default_availability: 0
|
||||
PublishClientTweetReply:
|
||||
default_availability: 0
|
||||
PublishClientTweetQuote:
|
||||
default_availability: 0
|
||||
PublishClientTweetRetweet:
|
||||
default_availability: 0
|
||||
PublishClientTweetClickReply:
|
||||
default_availability: 0
|
||||
PublishClientTweetClickQuote:
|
||||
default_availability: 0
|
||||
PublishClientTweetVideoPlayback25:
|
||||
default_availability: 0
|
||||
PublishClientTweetVideoPlayback50:
|
||||
default_availability: 0
|
||||
PublishClientTweetVideoPlayback75:
|
||||
default_availability: 0
|
||||
PublishClientTweetVideoPlayback95:
|
||||
default_availability: 0
|
||||
PublishClientTweetVideoPlayFromTap:
|
||||
default_availability: 0
|
||||
PublishClientTweetVideoQualityView:
|
||||
default_availability: 0
|
||||
PublishClientTweetVideoView:
|
||||
default_availability: 0
|
||||
PublishClientTweetVideoMrcView:
|
||||
default_availability: 0
|
||||
PublishClientTweetVideoViewThreshold:
|
||||
default_availability: 0
|
||||
PublishClientTweetVideoCtaUrlClick:
|
||||
default_availability: 0
|
||||
PublishClientTweetVideoCtaWatchClick:
|
||||
default_availability: 0
|
||||
PublishClientTweetUnretweet:
|
||||
default_availability: 0
|
||||
PublishClientTweetClickCaret:
|
||||
default_availability: 0
|
||||
PublishClientTweetPhotoExpand:
|
||||
default_availability: 0
|
||||
PublishClientTweetClickMentionScreenName:
|
||||
default_availability: 0
|
||||
PublishClientCardClick:
|
||||
default_availability: 0
|
||||
PublishClientCardOpenApp:
|
||||
default_availability: 0
|
||||
PublishClientCardAppInstallAttempt:
|
||||
default_availability: 0
|
||||
PublishClientPollCardVote:
|
||||
default_availability: 0
|
||||
PublishClientTweetProfileMentionClick:
|
||||
default_availability: 0
|
||||
PublishClientTweetClick:
|
||||
default_availability: 0
|
||||
PublishClientTopicFollow:
|
||||
default_availability: 0
|
||||
PublishClientTopicUnfollow:
|
||||
default_availability: 0
|
||||
PublishClientTopicNotInterestedIn:
|
||||
default_availability: 0
|
||||
PublishClientTopicUndoNotInterestedIn:
|
||||
default_availability: 0
|
||||
PublishClientTweetNotHelpful:
|
||||
default_availability: 0
|
||||
PublishClientTweetUndoNotHelpful:
|
||||
default_availability: 0
|
||||
PublishClientTweetReport:
|
||||
default_availability: 0
|
||||
PublishClientTweetNotInterestedIn:
|
||||
default_availability: 0
|
||||
PublishClientTweetUndoNotInterestedIn:
|
||||
default_availability: 0
|
||||
PublishClientTweetNotAboutTopic:
|
||||
default_availability: 0
|
||||
PublishClientTweetUndoNotAboutTopic:
|
||||
default_availability: 0
|
||||
PublishClientTweetNotRecent:
|
||||
default_availability: 0
|
||||
PublishClientTweetUndoNotRecent:
|
||||
default_availability: 0
|
||||
PublishClientTweetSeeFewer:
|
||||
default_availability: 0
|
||||
PublishClientTweetUndoSeeFewer:
|
||||
default_availability: 0
|
||||
PublishClientTweetNotRelevant:
|
||||
default_availability: 0
|
||||
PublishClientTweetUndoNotRelevant:
|
||||
default_availability: 0
|
||||
PublishClientProfileFollowAttempt:
|
||||
default_availability: 0
|
||||
PublishClientTweetFavoriteAttempt:
|
||||
default_availability: 0
|
||||
PublishClientTweetRetweetAttempt:
|
||||
default_availability: 0
|
||||
PublishClientTweetReplyAttempt:
|
||||
default_availability: 0
|
||||
PublishClientCTALoginClick:
|
||||
default_availability: 0
|
||||
PublishClientCTALoginStart:
|
||||
default_availability: 0
|
||||
PublishClientCTALoginSuccess:
|
||||
default_availability: 0
|
||||
PublishClientCTASignupClick:
|
||||
default_availability: 0
|
||||
PublishClientCTASignupSuccess:
|
||||
default_availability: 0
|
||||
PublishClientProfileBlock:
|
||||
default_availability: 0
|
||||
PublishClientProfileUnblock:
|
||||
default_availability: 0
|
||||
PublishClientProfileMute:
|
||||
default_availability: 0
|
||||
PublishClientProfileReport:
|
||||
default_availability: 0
|
||||
PublishClientProfileFollow:
|
||||
default_availability: 0
|
||||
PublishClientProfileClick:
|
||||
default_availability: 0
|
||||
PublishClientTweetFollowAuthor:
|
||||
default_availability: 0
|
||||
PublishClientTweetUnfollowAuthor:
|
||||
default_availability: 0
|
||||
PublishClientTweetBlockAuthor:
|
||||
default_availability: 0
|
||||
PublishClientTweetUnblockAuthor:
|
||||
default_availability: 0
|
||||
PublishClientTweetMuteAuthor:
|
||||
default_availability: 0
|
||||
PublishClientNotificationOpen:
|
||||
default_availability: 0
|
||||
PublishClientNotificationClick:
|
||||
default_availability: 0
|
||||
PublishClientNotificationSeeLessOften:
|
||||
default_availability: 0
|
||||
PublishClientNotificationDismiss:
|
||||
default_availability: 0
|
||||
PublishClientTypeaheadClick:
|
||||
default_availability: 0
|
||||
PublishClientFeedbackPromptSubmit:
|
||||
default_availability: 0
|
||||
PublishClientProfileShow:
|
||||
default_availability: 0
|
||||
PublishClientTweetV2Impression:
|
||||
default_availability: 0
|
||||
PublishClientTweetVideoFullscreenV2Impression:
|
||||
default_availability: 0
|
||||
PublishClientTweetImageFullscreenV2Impression:
|
||||
default_availability: 0
|
||||
PublishClientProfileV2Impression:
|
||||
default_availability: 0
|
||||
PublishClientTweetClickProfile:
|
||||
default_availability: 0
|
||||
PublishClientTweetClickShare:
|
||||
default_availability: 0
|
||||
PublishClientTweetShareViaCopyLink:
|
||||
default_availability: 0
|
||||
PublishClientTweetClickSendViaDirectMessage:
|
||||
default_availability: 0
|
||||
PublishClientTweetShareViaBookmark:
|
||||
default_availability: 0
|
||||
PublishClientTweetUnbookmark:
|
||||
default_availability: 0
|
||||
PublishClientTweetClickHashtag:
|
||||
default_availability: 0
|
||||
PublishClientTweetBookmark:
|
||||
default_availability: 0
|
||||
PublishClientTweetOpenLink:
|
||||
default_availability: 0
|
||||
PublishClientTweetTakeScreenshot:
|
||||
default_availability: 0
|
||||
PublishClientTweetVideoPlaybackStart:
|
||||
default_availability: 0
|
||||
PublishClientTweetVideoPlaybackComplete:
|
||||
default_availability: 0
|
||||
PublishClientTweetEmailClick:
|
||||
default_availability: 0
|
||||
PublishClientAppExit:
|
||||
default_availability: 0
|
||||
PublishClientTweetGalleryImpression:
|
||||
default_availability: 0
|
||||
PublishClientTweetDetailsImpression:
|
||||
default_availability: 0
|
||||
PublishClientTweetMomentImpression:
|
||||
default_availability: 0
|
||||
PublishServerUserCreate:
|
||||
default_availability: 0
|
||||
PublishServerUserUpdate:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetFav:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetUnfav:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetReply:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetRetweet:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetComposeTweet:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetBlockAuthor:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetUnblockAuthor:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetClick:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetReport:
|
||||
default_availability: 0
|
||||
PublishServerPromotedProfileFollow:
|
||||
default_availability: 0
|
||||
PublishServerPromotedProfileUnfollow:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetMuteAuthor:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetClickProfile:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetClickHashtag:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetOpenLink:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetCarouselSwipeNext:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetCarouselSwipePrevious:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetLingerImpressionShort:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetLingerImpressionMedium:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetLingerImpressionLong:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetClickSpotlight:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetViewSpotlight:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTrendView:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTrendClick:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetVideoPlayback25:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetVideoPlayback50:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetVideoPlayback75:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetVideoAdPlayback25:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetVideoAdPlayback50:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetVideoAdPlayback75:
|
||||
default_availability: 0
|
||||
PublishServerTweetVideoAdPlayback25:
|
||||
default_availability: 0
|
||||
PublishServerTweetVideoAdPlayback50:
|
||||
default_availability: 0
|
||||
PublishServerTweetVideoAdPlayback75:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetDismissWithoutReason:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetDismissUninteresting:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetDismissRepetitive:
|
||||
default_availability: 0
|
||||
PublishServerPromotedTweetDismissSpam:
|
||||
default_availability: 0
|
||||
PublishServerTweetArchiveFavorite:
|
||||
default_availability: 0
|
||||
PublishServerTweetUnarchiveFavorite:
|
||||
default_availability: 0
|
||||
PublishServerTweetArchiveRetweet:
|
||||
default_availability: 0
|
||||
PublishServerTweetUnarchiveRetweet:
|
||||
default_availability: 0
|
||||
RekeyUUAClientTweetRenderImpression:
|
||||
default_availability: 0
|
||||
RekeyUUAIesourceClientTweetRenderImpression:
|
||||
default_availability: 0
|
||||
EnrichmentPlannerSampling:
|
||||
default_availability: 0
|
||||
|
||||
85
unified_user_actions/service/src/main/resources/logback.xml
Normal file
85
unified_user_actions/service/src/main/resources/logback.xml
Normal file
@ -0,0 +1,85 @@
|
||||
<configuration>
|
||||
|
||||
<shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
|
||||
|
||||
<!-- ===================================================== -->
|
||||
<!-- JUL to SLF4J Bridging -->
|
||||
<!-- ===================================================== -->
|
||||
<contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
|
||||
<resetJUL>true</resetJUL>
|
||||
</contextListener>
|
||||
|
||||
<!-- ===================================================== -->
|
||||
<!-- Properties -->
|
||||
<!-- ===================================================== -->
|
||||
<property name="DEFAULT_SERVICE_PATTERN"
|
||||
value="%msg"/>
|
||||
|
||||
<!-- ===================================================== -->
|
||||
<!-- Secondary Appenders -->
|
||||
<!-- ===================================================== -->
|
||||
|
||||
<!-- Service Log (Rollover every 50MB, max 11 logs) -->
|
||||
<appender name="SERVICE" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${log.service.output}</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
|
||||
<fileNamePattern>${log.service.output}.%i</fileNamePattern>
|
||||
<minIndex>1</minIndex>
|
||||
<maxIndex>10</maxIndex>
|
||||
</rollingPolicy>
|
||||
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
|
||||
<maxFileSize>50MB</maxFileSize>
|
||||
</triggeringPolicy>
|
||||
<encoder>
|
||||
<pattern>%date %.-3level %logger ${DEFAULT_SERVICE_PATTERN}%n</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<!-- LogLens -->
|
||||
<appender name="LOGLENS" class="com.twitter.loglens.logback.LoglensAppender">
|
||||
<mdcAdditionalContext>false</mdcAdditionalContext>
|
||||
<index>${log.lens.index}</index>
|
||||
<tag>${log.lens.tag}/service</tag>
|
||||
<encoder>
|
||||
<pattern>${DEFAULT_SERVICE_PATTERN}</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<!-- ===================================================== -->
|
||||
<!-- Primary Async Appenders -->
|
||||
<!-- ===================================================== -->
|
||||
|
||||
<appender name="ASYNC-SERVICE" class="ch.qos.logback.classic.AsyncAppender">
|
||||
<appender-ref ref="SERVICE"/>
|
||||
</appender>
|
||||
|
||||
<appender name="ASYNC-LOGLENS" class="ch.qos.logback.classic.AsyncAppender">
|
||||
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
|
||||
<level>WARN</level>
|
||||
</filter>
|
||||
<appender-ref ref="LOGLENS"/>
|
||||
</appender>
|
||||
|
||||
<!-- ===================================================== -->
|
||||
<!-- Package Config -->
|
||||
<!-- ===================================================== -->
|
||||
|
||||
<!-- Root Config -->
|
||||
<root level="${log_level:-warn}">
|
||||
<appender-ref ref="ASYNC-SERVICE"/>
|
||||
<appender-ref ref="ASYNC-LOGLENS"/>
|
||||
</root>
|
||||
|
||||
<!-- Per-Package Config -->
|
||||
<logger name="com.twitter" level="info"/>
|
||||
<logger name="com.twitter.zookeeper.client.internal" level="warn"/>
|
||||
<logger name="com.twitter.zookeeper.client.internal.ClientCnxnSocket" level="error"/>
|
||||
<logger name="com.twitter.logging.ScribeHandler" level="warn"/>
|
||||
<logger name="com.twitter.finatra" level="info"/>
|
||||
|
||||
<logger name="org.apache.kafka" level="info"/>
|
||||
<logger name="org.apache.kafka.clients" level="info"/>
|
||||
<logger name="org.apache.kafka.clients.NetworkClient" level="warn"/>
|
||||
<logger name="org.apache.kafka.clients.consumer.internals" level="info"/>
|
||||
<logger name="org.apache.kafka.common.network" level="warn" />
|
||||
</configuration>
|
||||
390
unified_user_actions/service/src/main/scala/BUILD
Normal file
390
unified_user_actions/service/src/main/scala/BUILD
Normal file
@ -0,0 +1,390 @@
|
||||
jvm_binary(
|
||||
name = "uua-tls-favs-bin",
|
||||
basename = "uua-tls-favs-bin",
|
||||
main = "com.twitter.unified_user_actions.service.TlsFavsServiceMain",
|
||||
runtime_platform = "java11",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/ch/qos/logback:logback-classic",
|
||||
"loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback",
|
||||
"twitter-server-internal/src/main/scala",
|
||||
"twitter-server/logback-classic/src/main/scala",
|
||||
"unified_user_actions/service/src/main/resources",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:tls-favs",
|
||||
],
|
||||
)
|
||||
|
||||
jvm_app(
|
||||
name = "uua-tls-favs",
|
||||
archive = "zip",
|
||||
binary = ":uua-tls-favs-bin",
|
||||
bundles = [
|
||||
bundle(
|
||||
fileset = ["**/*"],
|
||||
owning_target = "unified_user_actions/service/src/main/resources:files",
|
||||
rel_path = "unified_user_actions/service/src/main/resources",
|
||||
),
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
)
|
||||
|
||||
jvm_binary(
|
||||
name = "uua-client-event-bin",
|
||||
basename = "uua-client-event-bin",
|
||||
main = "com.twitter.unified_user_actions.service.ClientEventServiceMain",
|
||||
runtime_platform = "java11",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/ch/qos/logback:logback-classic",
|
||||
"loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback",
|
||||
"twitter-server-internal/src/main/scala",
|
||||
"twitter-server/logback-classic/src/main/scala",
|
||||
"unified_user_actions/service/src/main/resources",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:client-event",
|
||||
],
|
||||
)
|
||||
|
||||
jvm_app(
|
||||
name = "uua-client-event",
|
||||
archive = "zip",
|
||||
binary = ":uua-client-event-bin",
|
||||
bundles = [
|
||||
bundle(
|
||||
fileset = ["**/*"],
|
||||
owning_target = "unified_user_actions/service/src/main/resources:files",
|
||||
rel_path = "unified_user_actions/service/src/main/resources",
|
||||
),
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
)
|
||||
|
||||
|
||||
jvm_binary(
|
||||
name = "uua-tweetypie-event-bin",
|
||||
basename = "uua-tweetypie-event-bin",
|
||||
main = "com.twitter.unified_user_actions.service.TweetypieEventServiceMain",
|
||||
runtime_platform = "java11",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/ch/qos/logback:logback-classic",
|
||||
"loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback",
|
||||
"twitter-server-internal/src/main/scala",
|
||||
"twitter-server/logback-classic/src/main/scala",
|
||||
"unified_user_actions/service/src/main/resources",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:tweetypie-event",
|
||||
],
|
||||
)
|
||||
|
||||
jvm_app(
|
||||
name = "uua-tweetypie-event",
|
||||
archive = "zip",
|
||||
binary = ":uua-tweetypie-event-bin",
|
||||
bundles = [
|
||||
bundle(
|
||||
fileset = ["**/*"],
|
||||
owning_target = "unified_user_actions/service/src/main/resources:files",
|
||||
rel_path = "unified_user_actions/service/src/main/resources",
|
||||
),
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
)
|
||||
|
||||
jvm_binary(
|
||||
name = "uua-social-graph-bin",
|
||||
basename = "uua-social-graph-bin",
|
||||
main = "com.twitter.unified_user_actions.service.SocialGraphServiceMain",
|
||||
runtime_platform = "java11",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/ch/qos/logback:logback-classic",
|
||||
"loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback",
|
||||
"twitter-server-internal/src/main/scala",
|
||||
"twitter-server/logback-classic/src/main/scala",
|
||||
"unified_user_actions/service/src/main/resources",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:social-graph-event",
|
||||
],
|
||||
)
|
||||
|
||||
jvm_app(
|
||||
name = "uua-social-graph",
|
||||
archive = "zip",
|
||||
binary = ":uua-social-graph-bin",
|
||||
bundles = [
|
||||
bundle(
|
||||
fileset = ["**/*"],
|
||||
owning_target = "unified_user_actions/service/src/main/resources:files",
|
||||
rel_path = "unified_user_actions/service/src/main/resources",
|
||||
),
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
)
|
||||
|
||||
jvm_binary(
|
||||
name = "uua-email-notification-event-bin",
|
||||
basename = "uua-email-notification-event-bin",
|
||||
main = "com.twitter.unified_user_actions.service.EmailNotificationEventServiceMain",
|
||||
runtime_platform = "java11",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/ch/qos/logback:logback-classic",
|
||||
"loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback",
|
||||
"twitter-server-internal/src/main/scala",
|
||||
"twitter-server/logback-classic/src/main/scala",
|
||||
"unified_user_actions/service/src/main/resources",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:email-notification-event",
|
||||
],
|
||||
)
|
||||
|
||||
jvm_app(
|
||||
name = "uua-email-notification-event",
|
||||
archive = "zip",
|
||||
binary = ":uua-email-notification-event-bin",
|
||||
bundles = [
|
||||
bundle(
|
||||
fileset = ["**/*"],
|
||||
owning_target = "unified_user_actions/service/src/main/resources:files",
|
||||
rel_path = "unified_user_actions/service/src/main/resources",
|
||||
),
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
)
|
||||
|
||||
jvm_binary(
|
||||
name = "uua-user-modification-bin",
|
||||
basename = "uua-user-modification-bin",
|
||||
main = "com.twitter.unified_user_actions.service.UserModificationServiceMain",
|
||||
runtime_platform = "java11",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/ch/qos/logback:logback-classic",
|
||||
"loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback",
|
||||
"twitter-server-internal/src/main/scala",
|
||||
"twitter-server/logback-classic/src/main/scala",
|
||||
"unified_user_actions/service/src/main/resources",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:user-modification-event",
|
||||
],
|
||||
)
|
||||
|
||||
jvm_app(
|
||||
name = "uua-user-modification",
|
||||
archive = "zip",
|
||||
binary = ":uua-user-modification-bin",
|
||||
bundles = [
|
||||
bundle(
|
||||
fileset = ["**/*"],
|
||||
owning_target = "unified_user_actions/service/src/main/resources:files",
|
||||
rel_path = "unified_user_actions/service/src/main/resources",
|
||||
),
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
)
|
||||
|
||||
jvm_binary(
|
||||
name = "uua-ads-callback-engagements-bin",
|
||||
basename = "uua-ads-callback-engagements-bin",
|
||||
main = "com.twitter.unified_user_actions.service.AdsCallbackEngagementsServiceMain",
|
||||
runtime_platform = "java11",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/ch/qos/logback:logback-classic",
|
||||
"loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback",
|
||||
"twitter-server-internal/src/main/scala",
|
||||
"twitter-server/logback-classic/src/main/scala",
|
||||
"unified_user_actions/service/src/main/resources",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:ads-callback-engagements",
|
||||
],
|
||||
)
|
||||
|
||||
jvm_app(
|
||||
name = "uua-ads-callback-engagements",
|
||||
archive = "zip",
|
||||
binary = ":uua-ads-callback-engagements-bin",
|
||||
bundles = [
|
||||
bundle(
|
||||
fileset = ["**/*"],
|
||||
owning_target = "unified_user_actions/service/src/main/resources:files",
|
||||
rel_path = "unified_user_actions/service/src/main/resources",
|
||||
),
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
)
|
||||
|
||||
jvm_binary(
|
||||
name = "uua-favorite-archival-events-bin",
|
||||
basename = "uua-favorite-archival-events-bin",
|
||||
main = "com.twitter.unified_user_actions.service.FavoriteArchivalEventsServiceMain",
|
||||
runtime_platform = "java11",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/ch/qos/logback:logback-classic",
|
||||
"loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback",
|
||||
"twitter-server-internal/src/main/scala",
|
||||
"twitter-server/logback-classic/src/main/scala",
|
||||
"unified_user_actions/service/src/main/resources",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:favorite-archival-events",
|
||||
],
|
||||
)
|
||||
|
||||
jvm_app(
|
||||
name = "uua-favorite-archival-events",
|
||||
archive = "zip",
|
||||
binary = ":uua-favorite-archival-events-bin",
|
||||
bundles = [
|
||||
bundle(
|
||||
fileset = ["**/*"],
|
||||
owning_target = "unified_user_actions/service/src/main/resources:files",
|
||||
rel_path = "unified_user_actions/service/src/main/resources",
|
||||
),
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
)
|
||||
|
||||
jvm_binary(
|
||||
name = "uua-retweet-archival-events-bin",
|
||||
basename = "uua-retweet-archival-events-bin",
|
||||
main = "com.twitter.unified_user_actions.service.RetweetArchivalEventsServiceMain",
|
||||
runtime_platform = "java11",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/ch/qos/logback:logback-classic",
|
||||
"loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback",
|
||||
"twitter-server-internal/src/main/scala",
|
||||
"twitter-server/logback-classic/src/main/scala",
|
||||
"unified_user_actions/service/src/main/resources",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:retweet-archival-events",
|
||||
],
|
||||
)
|
||||
|
||||
jvm_app(
|
||||
name = "uua-retweet-archival-events",
|
||||
archive = "zip",
|
||||
binary = ":uua-retweet-archival-events-bin",
|
||||
bundles = [
|
||||
bundle(
|
||||
fileset = ["**/*"],
|
||||
owning_target = "unified_user_actions/service/src/main/resources:files",
|
||||
rel_path = "unified_user_actions/service/src/main/resources",
|
||||
),
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
)
|
||||
|
||||
jvm_binary(
|
||||
name = "rekey-uua-bin",
|
||||
basename = "rekey-uua-bin",
|
||||
main = "com.twitter.unified_user_actions.service.RekeyUuaServiceMain",
|
||||
runtime_platform = "java11",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/ch/qos/logback:logback-classic",
|
||||
"loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback",
|
||||
"twitter-server-internal/src/main/scala",
|
||||
"twitter-server/logback-classic/src/main/scala",
|
||||
"unified_user_actions/service/src/main/resources",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:rekey-uua",
|
||||
],
|
||||
)
|
||||
|
||||
jvm_app(
|
||||
name = "rekey-uua",
|
||||
archive = "zip",
|
||||
binary = ":rekey-uua-bin",
|
||||
bundles = [
|
||||
bundle(
|
||||
fileset = ["**/*"],
|
||||
owning_target = "unified_user_actions/service/src/main/resources:files",
|
||||
rel_path = "unified_user_actions/service/src/main/resources",
|
||||
),
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
)
|
||||
|
||||
jvm_binary(
|
||||
name = "rekey-uua-iesource-bin",
|
||||
basename = "rekey-uua-iesource-bin",
|
||||
main = "com.twitter.unified_user_actions.service.RekeyUuaIesourceServiceMain",
|
||||
runtime_platform = "java11",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/ch/qos/logback:logback-classic",
|
||||
"loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback",
|
||||
"twitter-server-internal/src/main/scala",
|
||||
"twitter-server/logback-classic/src/main/scala",
|
||||
"unified_user_actions/service/src/main/resources",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:rekey-uua-iesource",
|
||||
],
|
||||
)
|
||||
|
||||
jvm_app(
|
||||
name = "rekey-uua-iesource",
|
||||
archive = "zip",
|
||||
binary = ":rekey-uua-iesource-bin",
|
||||
bundles = [
|
||||
bundle(
|
||||
fileset = ["**/*"],
|
||||
owning_target = "unified_user_actions/service/src/main/resources:files",
|
||||
rel_path = "unified_user_actions/service/src/main/resources",
|
||||
),
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
)
|
||||
|
||||
jvm_binary(
|
||||
name = "uua-enrichment-planner-bin",
|
||||
basename = "uua-enrichment-planner-bin",
|
||||
main = "com.twitter.unified_user_actions.service.EnrichmentPlannerServiceMain",
|
||||
runtime_platform = "java11",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/ch/qos/logback:logback-classic",
|
||||
"loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback",
|
||||
"twitter-server-internal/src/main/scala",
|
||||
"twitter-server/logback-classic/src/main/scala",
|
||||
"unified_user_actions/service/src/main/resources",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:enrichment-planner",
|
||||
],
|
||||
)
|
||||
|
||||
jvm_app(
|
||||
name = "uua-enrichment-planner",
|
||||
archive = "zip",
|
||||
binary = ":uua-enrichment-planner-bin",
|
||||
bundles = [
|
||||
bundle(
|
||||
fileset = ["**/*"],
|
||||
owning_target = "unified_user_actions/service/src/main/resources:files",
|
||||
rel_path = "unified_user_actions/service/src/main/resources",
|
||||
),
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
)
|
||||
|
||||
jvm_binary(
|
||||
name = "uua-enricher-bin",
|
||||
basename = "uua-enricher-bin",
|
||||
main = "com.twitter.unified_user_actions.service.EnricherServiceMain",
|
||||
runtime_platform = "java11",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/ch/qos/logback:logback-classic",
|
||||
"loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback",
|
||||
"twitter-server-internal/src/main/scala",
|
||||
"twitter-server/logback-classic/src/main/scala",
|
||||
"unified_user_actions/service/src/main/resources",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:enricher",
|
||||
],
|
||||
)
|
||||
|
||||
jvm_app(
|
||||
name = "uua-enricher",
|
||||
archive = "zip",
|
||||
binary = ":uua-enricher-bin",
|
||||
bundles = [
|
||||
bundle(
|
||||
fileset = ["**/*"],
|
||||
owning_target = "unified_user_actions/service/src/main/resources:files",
|
||||
rel_path = "unified_user_actions/service/src/main/resources",
|
||||
),
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
)
|
||||
@ -0,0 +1,25 @@
|
||||
package com.twitter.unified_user_actions.service
|
||||
|
||||
import com.twitter.ads.spendserver.thriftscala.SpendServerEvent
|
||||
import com.twitter.finatra.decider.modules.DeciderModule
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.inject.server.TwitterServer
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.unified_user_actions.service.module.KafkaProcessorAdsCallbackEngagementsModule
|
||||
|
||||
object AdsCallbackEngagementsServiceMain extends AdsCallbackEngagementsService
|
||||
|
||||
class AdsCallbackEngagementsService extends TwitterServer {
|
||||
override val modules = Seq(
|
||||
KafkaProcessorAdsCallbackEngagementsModule,
|
||||
DeciderModule
|
||||
)
|
||||
|
||||
override protected def setup(): Unit = {}
|
||||
|
||||
override protected def start(): Unit = {
|
||||
val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, SpendServerEvent]]
|
||||
closeOnExit(processor)
|
||||
processor.start()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,270 @@
|
||||
scala_library(
|
||||
name = "tls-favs",
|
||||
sources = ["TlsFavsService.scala"],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/org/apache/kafka:kafka-clients",
|
||||
"decider/src/main/scala",
|
||||
"finatra-internal/decider/src/main/scala",
|
||||
"finatra/inject/inject-server/src/main/scala/com/twitter/inject/server",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"kafka/libs/src/main/scala/com/twitter/kafka/client/processor",
|
||||
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:tls-favs",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-app/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "client-event",
|
||||
sources = ["ClientEventService.scala"],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/org/apache/kafka:kafka-clients",
|
||||
"decider/src/main/scala",
|
||||
"finatra-internal/decider/src/main/scala",
|
||||
"finatra/inject/inject-server/src/main/scala/com/twitter/inject/server",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"kafka/libs/src/main/scala/com/twitter/kafka/client/processor",
|
||||
"twadoop_config/configuration/log_categories/group/scribelib:client_event-scala",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:client-event",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-app/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "tweetypie-event",
|
||||
sources = ["TweetypieEventService.scala"],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/org/apache/kafka:kafka-clients",
|
||||
"decider/src/main/scala",
|
||||
"finatra-internal/decider/src/main/scala",
|
||||
"finatra/inject/inject-server/src/main/scala/com/twitter/inject/server",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"kafka/libs/src/main/scala/com/twitter/kafka/client/processor",
|
||||
"twadoop_config/configuration/log_categories/group/scribelib:client_event-scala",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:tweetypie-event",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-app/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "social-graph-event",
|
||||
sources = ["SocialGraphService.scala"],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/org/apache/kafka:kafka-clients",
|
||||
"decider/src/main/scala",
|
||||
"finatra-internal/decider/src/main/scala",
|
||||
"finatra/inject/inject-server/src/main/scala/com/twitter/inject/server",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"kafka/libs/src/main/scala/com/twitter/kafka/client/processor",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:social-graph-event",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-app/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "email-notification-event",
|
||||
sources = ["EmailNotificationEventService.scala"],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/org/apache/kafka:kafka-clients",
|
||||
"decider/src/main/scala",
|
||||
"finatra-internal/decider/src/main/scala",
|
||||
"finatra/inject/inject-server/src/main/scala/com/twitter/inject/server",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"kafka/libs/src/main/scala/com/twitter/kafka/client/processor",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:email-notification-event",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-app/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "user-modification-event",
|
||||
sources = ["UserModificationService.scala"],
|
||||
tags = [
|
||||
"bazel-compatible",
|
||||
"bazel-only",
|
||||
],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/org/apache/kafka:kafka-clients",
|
||||
"decider/src/main/scala",
|
||||
"finatra-internal/decider/src/main/scala",
|
||||
"finatra/inject/inject-server/src/main/scala/com/twitter/inject/server",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"kafka/libs/src/main/scala/com/twitter/kafka/client/processor",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:user-modification-event",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-app/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "ads-callback-engagements",
|
||||
sources = ["AdsCallbackEngagementsService.scala"],
|
||||
tags = [
|
||||
"bazel-compatible",
|
||||
"bazel-only",
|
||||
],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/org/apache/kafka:kafka-clients",
|
||||
"decider/src/main/scala",
|
||||
"finatra-internal/decider/src/main/scala",
|
||||
"finatra/inject/inject-server/src/main/scala/com/twitter/inject/server",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"kafka/libs/src/main/scala/com/twitter/kafka/client/processor",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:ads-callback-engagements",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-app/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "favorite-archival-events",
|
||||
sources = ["FavoriteArchivalEventsService.scala"],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/org/apache/kafka:kafka-clients",
|
||||
"decider/src/main/scala",
|
||||
"finatra-internal/decider/src/main/scala",
|
||||
"finatra/inject/inject-server/src/main/scala/com/twitter/inject/server",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"kafka/libs/src/main/scala/com/twitter/kafka/client/processor",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:favorite-archival-events",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-app/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "retweet-archival-events",
|
||||
sources = ["RetweetArchivalEventsService.scala"],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/org/apache/kafka:kafka-clients",
|
||||
"decider/src/main/scala",
|
||||
"finatra-internal/decider/src/main/scala",
|
||||
"finatra/inject/inject-server/src/main/scala/com/twitter/inject/server",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"kafka/libs/src/main/scala/com/twitter/kafka/client/processor",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:retweet-archival-events",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-app/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "rekey-uua",
|
||||
sources = ["RekeyUuaService.scala"],
|
||||
tags = [
|
||||
"bazel-compatible",
|
||||
"bazel-only",
|
||||
],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/org/apache/kafka:kafka-clients",
|
||||
"decider/src/main/scala",
|
||||
"finatra-internal/decider/src/main/scala",
|
||||
"finatra/inject/inject-server/src/main/scala/com/twitter/inject/server",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"kafka/libs/src/main/scala/com/twitter/kafka/client/processor",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:rekey-uua",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-app/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "rekey-uua-iesource",
|
||||
sources = ["RekeyUuaIesourceService.scala"],
|
||||
tags = [
|
||||
"bazel-compatible",
|
||||
"bazel-only",
|
||||
],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/org/apache/kafka:kafka-clients",
|
||||
"decider/src/main/scala",
|
||||
"finatra-internal/decider/src/main/scala",
|
||||
"finatra/inject/inject-server/src/main/scala/com/twitter/inject/server",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"kafka/libs/src/main/scala/com/twitter/kafka/client/processor",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:rekey-uua-iesource",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-app/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "enrichment-planner",
|
||||
sources = ["EnrichmentPlannerService.scala"],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"decider/src/main/scala",
|
||||
"finatra-internal/decider/src/main/scala",
|
||||
"finatra-internal/kafka-streams/kafka-streams/src/main/scala",
|
||||
"finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/producers",
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra-internal/mtls/src/main/scala",
|
||||
"kafka/finagle-kafka/finatra-kafka-streams/kafka-streams-static-partitioning/src/main/scala",
|
||||
"kafka/finagle-kafka/finatra-kafka-streams/kafka-streams/src/main/scala",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/driver",
|
||||
"unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator:noop",
|
||||
"unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner:default",
|
||||
"unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal:internal-scala",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "enricher",
|
||||
sources = ["EnricherService.scala"],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"finatra-internal/kafka-streams/kafka-streams/src/main/scala",
|
||||
"finatra-internal/mtls/src/main/scala",
|
||||
"finatra/inject/inject-server/src/main/scala/com/twitter/inject/server",
|
||||
"graphql/thrift/src/main/thrift/com/twitter/graphql:graphql-scala",
|
||||
"kafka/finagle-kafka/finatra-kafka-streams/kafka-streams-static-partitioning/src/main/scala",
|
||||
"kafka/finagle-kafka/finatra-kafka-streams/kafka-streams/src/main/scala",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/driver",
|
||||
"unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/graphql",
|
||||
"unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hydrator:default",
|
||||
"unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/partitioner:default",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:cache",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module:graphql-client",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-app/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,25 @@
|
||||
package com.twitter.unified_user_actions.service;
|
||||
|
||||
import com.twitter.finatra.decider.modules.DeciderModule
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.inject.server.TwitterServer
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.storage.behavioral_event.thriftscala.FlattenedEventLog
|
||||
import com.twitter.unified_user_actions.service.module.KafkaProcessorBehavioralClientEventModule
|
||||
|
||||
object BehavioralClientEventServiceMain extends BehavioralClientEventService
|
||||
|
||||
class BehavioralClientEventService extends TwitterServer {
|
||||
override val modules = Seq(
|
||||
KafkaProcessorBehavioralClientEventModule,
|
||||
DeciderModule
|
||||
)
|
||||
|
||||
override protected def setup(): Unit = {}
|
||||
|
||||
override protected def start(): Unit = {
|
||||
val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, FlattenedEventLog]]
|
||||
closeOnExit(processor)
|
||||
processor.start()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,23 @@
|
||||
package com.twitter.unified_user_actions.service
|
||||
|
||||
import com.twitter.clientapp.thriftscala.LogEvent
|
||||
import com.twitter.finatra.decider.modules.DeciderModule
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.inject.server.TwitterServer
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.unified_user_actions.service.module.KafkaProcessorClientEventModule
|
||||
|
||||
object ClientEventServiceMain extends ClientEventService
|
||||
|
||||
class ClientEventService extends TwitterServer {
|
||||
|
||||
override val modules = Seq(KafkaProcessorClientEventModule, DeciderModule)
|
||||
|
||||
override protected def setup(): Unit = {}
|
||||
|
||||
override protected def start(): Unit = {
|
||||
val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, LogEvent]]
|
||||
closeOnExit(processor)
|
||||
processor.start()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,26 @@
|
||||
package com.twitter.unified_user_actions.service
|
||||
|
||||
import com.twitter.finatra.decider.modules.DeciderModule
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.ibis.thriftscala.NotificationScribe
|
||||
import com.twitter.inject.server.TwitterServer
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.unified_user_actions.service.module.KafkaProcessorEmailNotificationEventModule
|
||||
|
||||
object EmailNotificationEventServiceMain extends EmailNotificationEventService
|
||||
|
||||
class EmailNotificationEventService extends TwitterServer {
|
||||
|
||||
override val modules = Seq(
|
||||
KafkaProcessorEmailNotificationEventModule,
|
||||
DeciderModule
|
||||
)
|
||||
|
||||
override protected def setup(): Unit = {}
|
||||
|
||||
override protected def start(): Unit = {
|
||||
val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, NotificationScribe]]
|
||||
closeOnExit(processor)
|
||||
processor.start()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,105 @@
|
||||
package com.twitter.unified_user_actions.service
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.conversions.StorageUnitOps._
|
||||
import com.twitter.dynmap.DynMap
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finatra.kafka.domain.AckMode
|
||||
import com.twitter.finatra.kafka.domain.KafkaGroupId
|
||||
import com.twitter.finatra.kafka.serde.ScalaSerdes
|
||||
import com.twitter.finatra.kafkastreams.config.KafkaStreamsConfig
|
||||
import com.twitter.finatra.kafkastreams.config.SecureKafkaStreamsConfig
|
||||
import com.twitter.finatra.kafkastreams.partitioning.StaticPartitioning
|
||||
import com.twitter.finatra.mtls.modules.ServiceIdentifierModule
|
||||
import com.twitter.finatra.kafkastreams.dsl.FinatraDslFlatMapAsync
|
||||
import com.twitter.graphql.thriftscala.GraphqlExecutionService
|
||||
import com.twitter.logging.Logging
|
||||
import com.twitter.unified_user_actions.enricher.driver.EnrichmentDriver
|
||||
import com.twitter.unified_user_actions.enricher.hcache.LocalCache
|
||||
import com.twitter.unified_user_actions.enricher.hydrator.DefaultHydrator
|
||||
import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentEnvelop
|
||||
import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentKey
|
||||
import com.twitter.unified_user_actions.enricher.partitioner.DefaultPartitioner
|
||||
import com.twitter.unified_user_actions.service.module.CacheModule
|
||||
import com.twitter.unified_user_actions.service.module.ClientIdModule
|
||||
import com.twitter.unified_user_actions.service.module.GraphqlClientProviderModule
|
||||
import com.twitter.util.Future
|
||||
import org.apache.kafka.common.record.CompressionType
|
||||
import org.apache.kafka.streams.StreamsBuilder
|
||||
import org.apache.kafka.streams.processor.RecordContext
|
||||
import org.apache.kafka.streams.processor.TopicNameExtractor
|
||||
import org.apache.kafka.streams.scala.kstream.Consumed
|
||||
import org.apache.kafka.streams.scala.kstream.Produced
|
||||
import com.twitter.unified_user_actions.enricher.driver.EnrichmentPlanUtils._
|
||||
|
||||
object EnricherServiceMain extends EnricherService
|
||||
|
||||
class EnricherService
|
||||
extends FinatraDslFlatMapAsync
|
||||
with StaticPartitioning
|
||||
with SecureKafkaStreamsConfig
|
||||
with Logging {
|
||||
val InputTopic = "unified_user_actions_keyed_dev"
|
||||
val OutputTopic = "unified_user_actions_enriched"
|
||||
|
||||
override val modules = Seq(
|
||||
CacheModule,
|
||||
ClientIdModule,
|
||||
GraphqlClientProviderModule,
|
||||
ServiceIdentifierModule
|
||||
)
|
||||
|
||||
override protected def configureKafkaStreams(builder: StreamsBuilder): Unit = {
|
||||
val graphqlClient = injector.instance[GraphqlExecutionService.FinagledClient]
|
||||
val localCache = injector.instance[LocalCache[EnrichmentKey, DynMap]]
|
||||
val statsReceiver = injector.instance[StatsReceiver]
|
||||
val driver = new EnrichmentDriver(
|
||||
finalOutputTopic = Some(OutputTopic),
|
||||
partitionedTopic = InputTopic,
|
||||
hydrator = new DefaultHydrator(
|
||||
cache = localCache,
|
||||
graphqlClient = graphqlClient,
|
||||
scopedStatsReceiver = statsReceiver.scope("DefaultHydrator")),
|
||||
partitioner = new DefaultPartitioner
|
||||
)
|
||||
|
||||
val kstream = builder.asScala
|
||||
.stream(InputTopic)(
|
||||
Consumed.`with`(ScalaSerdes.Thrift[EnrichmentKey], ScalaSerdes.Thrift[EnrichmentEnvelop]))
|
||||
.flatMapAsync[EnrichmentKey, EnrichmentEnvelop](
|
||||
commitInterval = 5.seconds,
|
||||
numWorkers = 10000
|
||||
) { (enrichmentKey: EnrichmentKey, enrichmentEnvelop: EnrichmentEnvelop) =>
|
||||
driver
|
||||
.execute(Some(enrichmentKey), Future.value(enrichmentEnvelop))
|
||||
.map(tuple => tuple._1.map(key => (key, tuple._2)).seq)
|
||||
}
|
||||
|
||||
val topicExtractor: TopicNameExtractor[EnrichmentKey, EnrichmentEnvelop] =
|
||||
(_: EnrichmentKey, envelop: EnrichmentEnvelop, _: RecordContext) =>
|
||||
envelop.plan.getLastCompletedStage.outputTopic.getOrElse(
|
||||
throw new IllegalStateException("Missing output topic in the last completed stage"))
|
||||
|
||||
kstream.to(topicExtractor)(
|
||||
Produced.`with`(ScalaSerdes.Thrift[EnrichmentKey], ScalaSerdes.Thrift[EnrichmentEnvelop]))
|
||||
}
|
||||
|
||||
override def streamsProperties(config: KafkaStreamsConfig): KafkaStreamsConfig =
|
||||
super
|
||||
.streamsProperties(config)
|
||||
.consumer.groupId(KafkaGroupId(applicationId()))
|
||||
.consumer.clientId(s"${applicationId()}-consumer")
|
||||
.consumer.requestTimeout(30.seconds)
|
||||
.consumer.sessionTimeout(30.seconds)
|
||||
.consumer.fetchMin(1.megabyte)
|
||||
.consumer.fetchMax(5.megabytes)
|
||||
.consumer.receiveBuffer(32.megabytes)
|
||||
.consumer.maxPollInterval(1.minute)
|
||||
.consumer.maxPollRecords(50000)
|
||||
.producer.clientId(s"${applicationId()}-producer")
|
||||
.producer.batchSize(16.kilobytes)
|
||||
.producer.bufferMemorySize(256.megabyte)
|
||||
.producer.requestTimeout(30.seconds)
|
||||
.producer.compressionType(CompressionType.LZ4)
|
||||
.producer.ackMode(AckMode.ALL)
|
||||
}
|
||||
@ -0,0 +1,187 @@
|
||||
package com.twitter.unified_user_actions.service
|
||||
|
||||
import com.twitter.app.Flag
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.conversions.StorageUnitOps._
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.decider.SimpleRecipient
|
||||
import com.twitter.finatra.decider.modules.DeciderModule
|
||||
import com.twitter.finatra.kafka.domain.AckMode
|
||||
import com.twitter.finatra.kafka.domain.KafkaGroupId
|
||||
import com.twitter.finatra.kafka.domain.KafkaTopic
|
||||
import com.twitter.finatra.kafka.producers.FinagleKafkaProducerConfig
|
||||
import com.twitter.finatra.kafka.producers.KafkaProducerConfig
|
||||
import com.twitter.finatra.kafka.producers.TwitterKafkaProducerConfig
|
||||
import com.twitter.finatra.kafka.serde.ScalaSerdes
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.finatra.kafka.serde.UnKeyedSerde
|
||||
import com.twitter.finatra.kafkastreams.config.KafkaStreamsConfig
|
||||
import com.twitter.finatra.kafkastreams.config.SecureKafkaStreamsConfig
|
||||
import com.twitter.finatra.kafkastreams.dsl.FinatraDslToCluster
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.unified_user_actions.enricher.driver.EnrichmentDriver
|
||||
import com.twitter.unified_user_actions.enricher.hydrator.NoopHydrator
|
||||
import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentEnvelop
|
||||
import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentInstruction.NotificationTweetEnrichment
|
||||
import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentInstruction.TweetEnrichment
|
||||
import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentKey
|
||||
import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentPlan
|
||||
import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentStage
|
||||
import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentStageStatus
|
||||
import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentStageType
|
||||
import com.twitter.unified_user_actions.enricher.partitioner.DefaultPartitioner
|
||||
import com.twitter.unified_user_actions.enricher.partitioner.DefaultPartitioner.NullKey
|
||||
import com.twitter.unified_user_actions.thriftscala.Item
|
||||
import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction
|
||||
import com.twitter.util.Await
|
||||
import com.twitter.util.Future
|
||||
import org.apache.kafka.common.record.CompressionType
|
||||
import org.apache.kafka.streams.StreamsBuilder
|
||||
import org.apache.kafka.streams.scala.kstream.Consumed
|
||||
import org.apache.kafka.streams.scala.kstream.KStream
|
||||
import org.apache.kafka.streams.scala.kstream.Produced
|
||||
object EnrichmentPlannerServiceMain extends EnrichmentPlannerService {
|
||||
val ApplicationId = "uua-enrichment-planner"
|
||||
val InputTopic = "unified_user_actions"
|
||||
val OutputPartitionedTopic = "unified_user_actions_keyed_dev"
|
||||
val SamplingDecider = "EnrichmentPlannerSampling"
|
||||
}
|
||||
|
||||
/**
|
||||
* This service is the first step (planner) of the UUA Enrichment process.
|
||||
* It does the following:
|
||||
* 1. Read Prod UUA topic unified_user_actions from the Prod cluster and write to (see below) either Prod cluster (prod) or Dev cluster (dev/staging)
|
||||
* 2. For the write, it optionally randomly downsample the events when publishing, controlled by a Decider
|
||||
* 3. The output's key would be the first step of the repartitioning, most likely the EnrichmentKey of the Tweet type.
|
||||
*/
|
||||
class EnrichmentPlannerService extends FinatraDslToCluster with SecureKafkaStreamsConfig {
|
||||
import EnrichmentPlannerServiceMain._
|
||||
|
||||
val kafkaOutputCluster: Flag[String] = flag(
|
||||
name = "kafka.output.server",
|
||||
default = "",
|
||||
help =
|
||||
"""The output Kafka cluster.
|
||||
|This is needed since we read from a cluster and potentially output to a different cluster.
|
||||
|""".stripMargin
|
||||
)
|
||||
|
||||
val kafkaOutputEnableTls: Flag[Boolean] = flag(
|
||||
name = "kafka.output.enable.tls",
|
||||
default = true,
|
||||
help = ""
|
||||
)
|
||||
|
||||
override val modules: Seq[TwitterModule] = Seq(
|
||||
DeciderModule
|
||||
)
|
||||
|
||||
override protected def configureKafkaStreams(builder: StreamsBuilder): Unit = {
|
||||
val decider = injector.instance[Decider]
|
||||
val driver = new EnrichmentDriver(
|
||||
finalOutputTopic = NoopHydrator.OutputTopic,
|
||||
partitionedTopic = OutputPartitionedTopic,
|
||||
hydrator = new NoopHydrator,
|
||||
partitioner = new DefaultPartitioner)
|
||||
|
||||
val builderWithoutOutput = builder.asScala
|
||||
.stream(InputTopic)(Consumed.`with`(UnKeyedSerde, ScalaSerdes.Thrift[UnifiedUserAction]))
|
||||
// this maps and filters out the nil envelop before further processing
|
||||
.flatMapValues { uua =>
|
||||
(uua.item match {
|
||||
case Item.TweetInfo(_) =>
|
||||
Some(EnrichmentEnvelop(
|
||||
envelopId = uua.hashCode.toLong,
|
||||
uua = uua,
|
||||
plan = EnrichmentPlan(Seq(
|
||||
EnrichmentStage(
|
||||
status = EnrichmentStageStatus.Initialized,
|
||||
stageType = EnrichmentStageType.Repartition,
|
||||
instructions = Seq(TweetEnrichment)
|
||||
),
|
||||
EnrichmentStage(
|
||||
status = EnrichmentStageStatus.Initialized,
|
||||
stageType = EnrichmentStageType.Hydration,
|
||||
instructions = Seq(TweetEnrichment)
|
||||
),
|
||||
))
|
||||
))
|
||||
case Item.NotificationInfo(_) =>
|
||||
Some(EnrichmentEnvelop(
|
||||
envelopId = uua.hashCode.toLong,
|
||||
uua = uua,
|
||||
plan = EnrichmentPlan(Seq(
|
||||
EnrichmentStage(
|
||||
status = EnrichmentStageStatus.Initialized,
|
||||
stageType = EnrichmentStageType.Repartition,
|
||||
instructions = Seq(NotificationTweetEnrichment)
|
||||
),
|
||||
EnrichmentStage(
|
||||
status = EnrichmentStageStatus.Initialized,
|
||||
stageType = EnrichmentStageType.Hydration,
|
||||
instructions = Seq(NotificationTweetEnrichment)
|
||||
),
|
||||
))
|
||||
))
|
||||
case _ => None
|
||||
}).seq
|
||||
}
|
||||
// execute our driver logics
|
||||
.flatMap((_: UnKeyed, envelop: EnrichmentEnvelop) => {
|
||||
// flatMap and Await.result is used here because our driver interface allows for
|
||||
// both synchronous (repartition logic) and async operations (hydration logic), but in here
|
||||
// we purely just need to repartition synchronously, and thus the flatMap + Await.result
|
||||
// is used to simplify and make testing much easier.
|
||||
val (keyOpt, value) = Await.result(driver.execute(NullKey, Future.value(envelop)))
|
||||
keyOpt.map(key => (key, value)).seq
|
||||
})
|
||||
// then finally we sample based on the output keys
|
||||
.filter((key, _) =>
|
||||
decider.isAvailable(feature = SamplingDecider, Some(SimpleRecipient(key.id))))
|
||||
|
||||
configureOutput(builderWithoutOutput)
|
||||
}
|
||||
|
||||
private def configureOutput(kstream: KStream[EnrichmentKey, EnrichmentEnvelop]): Unit = {
|
||||
if (kafkaOutputCluster().nonEmpty && kafkaOutputCluster() != bootstrapServer()) {
|
||||
kstream.toCluster(
|
||||
cluster = kafkaOutputCluster(),
|
||||
topic = KafkaTopic(OutputPartitionedTopic),
|
||||
clientId = s"$ApplicationId-output-producer",
|
||||
kafkaProducerConfig =
|
||||
if (kafkaOutputEnableTls())
|
||||
FinagleKafkaProducerConfig[EnrichmentKey, EnrichmentEnvelop](kafkaProducerConfig =
|
||||
KafkaProducerConfig(TwitterKafkaProducerConfig().requestTimeout(1.minute).configMap))
|
||||
else
|
||||
FinagleKafkaProducerConfig[EnrichmentKey, EnrichmentEnvelop](
|
||||
kafkaProducerConfig = KafkaProducerConfig()
|
||||
.requestTimeout(1.minute)),
|
||||
statsReceiver = statsReceiver,
|
||||
commitInterval = 15.seconds
|
||||
)(Produced.`with`(ScalaSerdes.Thrift[EnrichmentKey], ScalaSerdes.Thrift[EnrichmentEnvelop]))
|
||||
} else {
|
||||
kstream.to(OutputPartitionedTopic)(
|
||||
Produced.`with`(ScalaSerdes.Thrift[EnrichmentKey], ScalaSerdes.Thrift[EnrichmentEnvelop]))
|
||||
}
|
||||
}
|
||||
|
||||
override def streamsProperties(config: KafkaStreamsConfig): KafkaStreamsConfig = {
|
||||
super
|
||||
.streamsProperties(config)
|
||||
.consumer.groupId(KafkaGroupId(ApplicationId))
|
||||
.consumer.clientId(s"$ApplicationId-consumer")
|
||||
.consumer.requestTimeout(30.seconds)
|
||||
.consumer.sessionTimeout(30.seconds)
|
||||
.consumer.fetchMin(1.megabyte)
|
||||
.consumer.fetchMax(5.megabyte)
|
||||
.consumer.receiveBuffer(32.megabytes)
|
||||
.consumer.maxPollInterval(1.minute)
|
||||
.consumer.maxPollRecords(50000)
|
||||
.producer.clientId(s"$ApplicationId-producer")
|
||||
.producer.batchSize(16.kilobytes)
|
||||
.producer.bufferMemorySize(256.megabyte)
|
||||
.producer.requestTimeout(30.seconds)
|
||||
.producer.compressionType(CompressionType.LZ4)
|
||||
.producer.ackMode(AckMode.ALL)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,26 @@
|
||||
package com.twitter.unified_user_actions.service
|
||||
|
||||
import com.twitter.finatra.decider.modules.DeciderModule
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.inject.server.TwitterServer
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.timelineservice.fanout.thriftscala.FavoriteArchivalEvent
|
||||
import com.twitter.unified_user_actions.service.module.KafkaProcessorFavoriteArchivalEventsModule
|
||||
|
||||
object FavoriteArchivalEventsServiceMain extends FavoriteArchivalEventsService
|
||||
|
||||
class FavoriteArchivalEventsService extends TwitterServer {
|
||||
|
||||
override val modules = Seq(
|
||||
KafkaProcessorFavoriteArchivalEventsModule,
|
||||
DeciderModule
|
||||
)
|
||||
|
||||
override protected def setup(): Unit = {}
|
||||
|
||||
override protected def start(): Unit = {
|
||||
val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, FavoriteArchivalEvent]]
|
||||
closeOnExit(processor)
|
||||
processor.start()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,26 @@
|
||||
package com.twitter.unified_user_actions.service
|
||||
|
||||
import com.twitter.finatra.decider.modules.DeciderModule
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.iesource.thriftscala.InteractionEvent
|
||||
import com.twitter.inject.server.TwitterServer
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.unified_user_actions.service.module.KafkaProcessorRekeyUuaIesourceModule
|
||||
|
||||
object RekeyUuaIesourceServiceMain extends RekeyUuaIesourceService
|
||||
|
||||
class RekeyUuaIesourceService extends TwitterServer {
|
||||
|
||||
override val modules = Seq(
|
||||
KafkaProcessorRekeyUuaIesourceModule,
|
||||
DeciderModule
|
||||
)
|
||||
|
||||
override protected def setup(): Unit = {}
|
||||
|
||||
override protected def start(): Unit = {
|
||||
val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, InteractionEvent]]
|
||||
closeOnExit(processor)
|
||||
processor.start()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,26 @@
|
||||
package com.twitter.unified_user_actions.service
|
||||
|
||||
import com.twitter.finatra.decider.modules.DeciderModule
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.inject.server.TwitterServer
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.unified_user_actions.service.module.KafkaProcessorRekeyUuaModule
|
||||
import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction
|
||||
|
||||
object RekeyUuaServiceMain extends RekeyUuaService
|
||||
|
||||
class RekeyUuaService extends TwitterServer {
|
||||
|
||||
override val modules = Seq(
|
||||
KafkaProcessorRekeyUuaModule,
|
||||
DeciderModule
|
||||
)
|
||||
|
||||
override protected def setup(): Unit = {}
|
||||
|
||||
override protected def start(): Unit = {
|
||||
val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, UnifiedUserAction]]
|
||||
closeOnExit(processor)
|
||||
processor.start()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,26 @@
|
||||
package com.twitter.unified_user_actions.service
|
||||
|
||||
import com.twitter.finatra.decider.modules.DeciderModule
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.inject.server.TwitterServer
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.tweetypie.thriftscala.RetweetArchivalEvent
|
||||
import com.twitter.unified_user_actions.service.module.KafkaProcessorRetweetArchivalEventsModule
|
||||
|
||||
object RetweetArchivalEventsServiceMain extends RetweetArchivalEventsService
|
||||
|
||||
class RetweetArchivalEventsService extends TwitterServer {
|
||||
|
||||
override val modules = Seq(
|
||||
KafkaProcessorRetweetArchivalEventsModule,
|
||||
DeciderModule
|
||||
)
|
||||
|
||||
override protected def setup(): Unit = {}
|
||||
|
||||
override protected def start(): Unit = {
|
||||
val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, RetweetArchivalEvent]]
|
||||
closeOnExit(processor)
|
||||
processor.start()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,25 @@
|
||||
package com.twitter.unified_user_actions.service
|
||||
|
||||
import com.twitter.finatra.decider.modules.DeciderModule
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.inject.server.TwitterServer
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.socialgraph.thriftscala.WriteEvent
|
||||
import com.twitter.unified_user_actions.service.module.KafkaProcessorSocialGraphModule
|
||||
|
||||
object SocialGraphServiceMain extends SocialGraphService
|
||||
|
||||
class SocialGraphService extends TwitterServer {
|
||||
override val modules = Seq(
|
||||
KafkaProcessorSocialGraphModule,
|
||||
DeciderModule
|
||||
)
|
||||
|
||||
override protected def setup(): Unit = {}
|
||||
|
||||
override protected def start(): Unit = {
|
||||
val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, WriteEvent]]
|
||||
closeOnExit(processor)
|
||||
processor.start()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,26 @@
|
||||
package com.twitter.unified_user_actions.service
|
||||
|
||||
import com.twitter.finatra.decider.modules.DeciderModule
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.inject.server.TwitterServer
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.timelineservice.thriftscala.ContextualizedFavoriteEvent
|
||||
import com.twitter.unified_user_actions.service.module.KafkaProcessorTlsFavsModule
|
||||
|
||||
object TlsFavsServiceMain extends TlsFavsService
|
||||
|
||||
class TlsFavsService extends TwitterServer {
|
||||
|
||||
override val modules = Seq(
|
||||
KafkaProcessorTlsFavsModule,
|
||||
DeciderModule
|
||||
)
|
||||
|
||||
override protected def setup(): Unit = {}
|
||||
|
||||
override protected def start(): Unit = {
|
||||
val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, ContextualizedFavoriteEvent]]
|
||||
closeOnExit(processor)
|
||||
processor.start()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,27 @@
|
||||
package com.twitter.unified_user_actions.service
|
||||
|
||||
import com.twitter.finatra.decider.modules.DeciderModule
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.inject.server.TwitterServer
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.tweetypie.thriftscala.TweetEvent
|
||||
import com.twitter.unified_user_actions.service.module.KafkaProcessorTweetypieEventModule
|
||||
|
||||
object TweetypieEventServiceMain extends TweetypieEventService
|
||||
|
||||
class TweetypieEventService extends TwitterServer {
|
||||
|
||||
override val modules = Seq(
|
||||
KafkaProcessorTweetypieEventModule,
|
||||
DeciderModule
|
||||
)
|
||||
|
||||
override protected def setup(): Unit = {}
|
||||
|
||||
override protected def start(): Unit = {
|
||||
val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, TweetEvent]]
|
||||
closeOnExit(processor)
|
||||
processor.start()
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,25 @@
|
||||
package com.twitter.unified_user_actions.service
|
||||
|
||||
import com.twitter.finatra.decider.modules.DeciderModule
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.gizmoduck.thriftscala.UserModification
|
||||
import com.twitter.inject.server.TwitterServer
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.unified_user_actions.service.module.KafkaProcessorUserModificationModule
|
||||
|
||||
object UserModificationServiceMain extends UserModificationService
|
||||
|
||||
class UserModificationService extends TwitterServer {
|
||||
override val modules = Seq(
|
||||
KafkaProcessorUserModificationModule,
|
||||
DeciderModule
|
||||
)
|
||||
|
||||
override protected def setup(): Unit = {}
|
||||
|
||||
override protected def start(): Unit = {
|
||||
val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, UserModification]]
|
||||
closeOnExit(processor)
|
||||
processor.start()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,482 @@
|
||||
scala_library(
|
||||
name = "decider-utils",
|
||||
sources = [
|
||||
"DeciderUtils.scala",
|
||||
"TopicsMapping.scala",
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"decider/src/main/scala",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "base",
|
||||
sources = [
|
||||
"FlagsModule.scala",
|
||||
"KafkaProcessorProvider.scala",
|
||||
"ZoneFiltering.scala",
|
||||
],
|
||||
tags = [
|
||||
"bazel-compatible",
|
||||
"bazel-only",
|
||||
],
|
||||
dependencies = [
|
||||
":decider-utils",
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"3rdparty/jvm/org/apache/kafka:kafka-clients",
|
||||
"finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers",
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules",
|
||||
"finatra/inject/inject-thrift-client/src/main/scala",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"kafka/libs/src/main/scala/com/twitter/kafka/client/processor",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter:base",
|
||||
"unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-core:scala",
|
||||
"util/util-core/src/main/scala/com/twitter/conversions",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "tls-favs",
|
||||
sources = [
|
||||
"FlagsModule.scala",
|
||||
"KafkaProcessorProvider.scala",
|
||||
"KafkaProcessorTlsFavsModule.scala",
|
||||
"ZoneFiltering.scala",
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
":decider-utils",
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"3rdparty/jvm/org/apache/kafka:kafka-clients",
|
||||
"finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers",
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules",
|
||||
"finatra/inject/inject-thrift-client/src/main/scala",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"kafka/libs/src/main/scala/com/twitter/kafka/client/processor",
|
||||
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tls_favs_event",
|
||||
"unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-core:scala",
|
||||
"util/util-core/src/main/scala/com/twitter/conversions",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "client-event",
|
||||
sources = [
|
||||
"FlagsModule.scala",
|
||||
"KafkaProcessorClientEventModule.scala",
|
||||
"KafkaProcessorProvider.scala",
|
||||
"TopicsMapping.scala",
|
||||
"ZoneFiltering.scala",
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
":decider-utils",
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"3rdparty/jvm/org/apache/kafka:kafka-clients",
|
||||
"finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers",
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules",
|
||||
"finatra/inject/inject-thrift-client/src/main/scala",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"kafka/libs/src/main/scala/com/twitter/kafka/client/processor",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/client_event",
|
||||
"unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-core:scala",
|
||||
"util/util-core/src/main/scala/com/twitter/conversions",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
scala_library(
|
||||
name = "tweetypie-event",
|
||||
sources = [
|
||||
"FlagsModule.scala",
|
||||
"KafkaProcessorProvider.scala",
|
||||
"KafkaProcessorTweetypieEventModule.scala",
|
||||
"ZoneFiltering.scala",
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
":decider-utils",
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"3rdparty/jvm/org/apache/kafka:kafka-clients",
|
||||
"finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers",
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules",
|
||||
"finatra/inject/inject-thrift-client/src/main/scala",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"kafka/libs/src/main/scala/com/twitter/kafka/client/processor",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/tweetypie_event",
|
||||
"unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-core:scala",
|
||||
"util/util-core/src/main/scala/com/twitter/conversions",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "social-graph-event",
|
||||
sources = [
|
||||
"FlagsModule.scala",
|
||||
"KafkaProcessorProvider.scala",
|
||||
"KafkaProcessorSocialGraphModule.scala",
|
||||
"ZoneFiltering.scala",
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
":decider-utils",
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"3rdparty/jvm/org/apache/kafka:kafka-clients",
|
||||
"finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers",
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules",
|
||||
"finatra/inject/inject-thrift-client/src/main/scala",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"kafka/libs/src/main/scala/com/twitter/kafka/client/processor",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/social_graph_event",
|
||||
"unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-core:scala",
|
||||
"util/util-core/src/main/scala/com/twitter/conversions",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "email-notification-event",
|
||||
sources = [
|
||||
"FlagsModule.scala",
|
||||
"KafkaProcessorEmailNotificationEventModule.scala",
|
||||
"KafkaProcessorProvider.scala",
|
||||
"ZoneFiltering.scala",
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
":decider-utils",
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"3rdparty/jvm/org/apache/kafka:kafka-clients",
|
||||
"finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers",
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules",
|
||||
"finatra/inject/inject-thrift-client/src/main/scala",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"kafka/libs/src/main/scala/com/twitter/kafka/client/processor",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/email_notification_event",
|
||||
"unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-core:scala",
|
||||
"util/util-core/src/main/scala/com/twitter/conversions",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "user-modification-event",
|
||||
sources = [
|
||||
"FlagsModule.scala",
|
||||
"KafkaProcessorProvider.scala",
|
||||
"KafkaProcessorUserModificationModule.scala",
|
||||
"ZoneFiltering.scala",
|
||||
],
|
||||
tags = [
|
||||
"bazel-compatible",
|
||||
"bazel-only",
|
||||
],
|
||||
dependencies = [
|
||||
":decider-utils",
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"3rdparty/jvm/org/apache/kafka:kafka-clients",
|
||||
"finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers",
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules",
|
||||
"finatra/inject/inject-thrift-client/src/main/scala",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"kafka/libs/src/main/scala/com/twitter/kafka/client/processor",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/user_modification_event",
|
||||
"unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-core:scala",
|
||||
"util/util-core/src/main/scala/com/twitter/conversions",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "ads-callback-engagements",
|
||||
sources = [
|
||||
"FlagsModule.scala",
|
||||
"KafkaProcessorAdsCallbackEngagementsModule.scala",
|
||||
"KafkaProcessorProvider.scala",
|
||||
"ZoneFiltering.scala",
|
||||
],
|
||||
tags = [
|
||||
"bazel-compatible",
|
||||
"bazel-only",
|
||||
],
|
||||
dependencies = [
|
||||
":decider-utils",
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"3rdparty/jvm/org/apache/kafka:kafka-clients",
|
||||
"finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers",
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules",
|
||||
"finatra/inject/inject-thrift-client/src/main/scala",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"kafka/libs/src/main/scala/com/twitter/kafka/client/processor",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/ads_callback_engagements",
|
||||
"unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-core:scala",
|
||||
"util/util-core/src/main/scala/com/twitter/conversions",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "favorite-archival-events",
|
||||
sources = [
|
||||
"FlagsModule.scala",
|
||||
"KafkaProcessorFavoriteArchivalEventsModule.scala",
|
||||
"KafkaProcessorProvider.scala",
|
||||
"ZoneFiltering.scala",
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
":decider-utils",
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"3rdparty/jvm/org/apache/kafka:kafka-clients",
|
||||
"finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers",
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules",
|
||||
"finatra/inject/inject-thrift-client/src/main/scala",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"kafka/libs/src/main/scala/com/twitter/kafka/client/processor",
|
||||
"src/thrift/com/twitter/timelineservice/server/internal:thrift-scala",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/favorite_archival_events",
|
||||
"unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-core:scala",
|
||||
"util/util-core/src/main/scala/com/twitter/conversions",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "retweet-archival-events",
|
||||
sources = [
|
||||
"FlagsModule.scala",
|
||||
"KafkaProcessorProvider.scala",
|
||||
"KafkaProcessorRetweetArchivalEventsModule.scala",
|
||||
"ZoneFiltering.scala",
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
":decider-utils",
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"3rdparty/jvm/org/apache/kafka:kafka-clients",
|
||||
"finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers",
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules",
|
||||
"finatra/inject/inject-thrift-client/src/main/scala",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"kafka/libs/src/main/scala/com/twitter/kafka/client/processor",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/retweet_archival_events",
|
||||
"unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-core:scala",
|
||||
"util/util-core/src/main/scala/com/twitter/conversions",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "rekey-uua",
|
||||
sources = [
|
||||
"FlagsModule.scala",
|
||||
"KafkaProcessorProvider.scala",
|
||||
"KafkaProcessorRekeyUuaModule.scala",
|
||||
"ZoneFiltering.scala",
|
||||
],
|
||||
tags = [
|
||||
"bazel-compatible",
|
||||
"bazel-only",
|
||||
],
|
||||
dependencies = [
|
||||
":decider-utils",
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"3rdparty/jvm/org/apache/kafka:kafka-clients",
|
||||
"finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers",
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules",
|
||||
"finatra/inject/inject-thrift-client/src/main/scala",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"kafka/libs/src/main/scala/com/twitter/kafka/client/processor",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates",
|
||||
"unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-core:scala",
|
||||
"util/util-core/src/main/scala/com/twitter/conversions",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "rekey-uua-iesource",
|
||||
sources = [
|
||||
"FlagsModule.scala",
|
||||
"KafkaProcessorProvider.scala",
|
||||
"KafkaProcessorRekeyUuaIesourceModule.scala",
|
||||
"ZoneFiltering.scala",
|
||||
],
|
||||
tags = [
|
||||
"bazel-compatible",
|
||||
"bazel-only",
|
||||
],
|
||||
dependencies = [
|
||||
":decider-utils",
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"3rdparty/jvm/org/apache/kafka:kafka-clients",
|
||||
"finatra-internal/kafka/src/main/scala/com/twitter/finatra/kafka/consumers",
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules",
|
||||
"finatra/inject/inject-thrift-client/src/main/scala",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/main/scala",
|
||||
"kafka/libs/src/main/scala/com/twitter/kafka/client/processor",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/adapter/src/main/scala/com/twitter/unified_user_actions/adapter/uua_aggregates",
|
||||
"unified_user_actions/kafka/src/main/scala/com/twitter/unified_user_actions/kafka",
|
||||
"unified_user_actions/thrift/src/main/thrift/com/twitter/unified_user_actions:unified_user_actions-scala",
|
||||
"util/util-core:scala",
|
||||
"util/util-core/src/main/scala/com/twitter/conversions",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "graphql-client",
|
||||
sources = [
|
||||
"ClientIdModule.scala",
|
||||
"GraphqlClientProviderModule.scala",
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication",
|
||||
"finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/client",
|
||||
"finagle/finagle-thriftmux/src/main/scala",
|
||||
"finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations",
|
||||
"finatra/inject/inject-modules/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules",
|
||||
"graphql/thrift/src/main/thrift/com/twitter/graphql:graphql-scala",
|
||||
"twitter-server/server/src/main/scala",
|
||||
],
|
||||
)
|
||||
|
||||
scala_library(
|
||||
name = "cache",
|
||||
sources = [
|
||||
"CacheModule.scala",
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/guava",
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"featureswitches/dynmap/src/main/scala/com/twitter/dynmap:dynmap-core",
|
||||
"finatra/inject/inject-app/src/main/java/com/twitter/inject/annotations",
|
||||
"finatra/inject/inject-modules/src/main/scala",
|
||||
"finatra/inject/inject-modules/src/main/scala/com/twitter/inject/modules",
|
||||
"graphql/thrift/src/main/thrift/com/twitter/graphql:graphql-scala",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"unified_user_actions/enricher/src/main/scala/com/twitter/unified_user_actions/enricher/hcache",
|
||||
"unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal:internal-scala",
|
||||
"util/util-cache-guava/src/main/scala",
|
||||
"util/util-cache/src/main/scala",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,48 @@
|
||||
package com.twitter.unified_user_actions.service.module
|
||||
|
||||
import com.google.common.cache.CacheBuilder
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.dynmap.DynMap
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.inject.annotations.Flag
|
||||
import com.twitter.unified_user_actions.enricher.hcache.LocalCache
|
||||
import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentKey
|
||||
import com.twitter.util.Future
|
||||
import java.util.concurrent.TimeUnit
|
||||
import javax.inject.Singleton
|
||||
|
||||
object CacheModule extends TwitterModule {
|
||||
private final val localCacheTtlFlagName = "local.cache.ttl.seconds"
|
||||
private final val localCacheMaxSizeFlagName = "local.cache.max.size"
|
||||
|
||||
flag[Long](
|
||||
name = localCacheTtlFlagName,
|
||||
default = 1800L,
|
||||
help = "Local Cache's TTL in seconds"
|
||||
)
|
||||
|
||||
flag[Long](
|
||||
name = localCacheMaxSizeFlagName,
|
||||
default = 1000L,
|
||||
help = "Local Cache's max size"
|
||||
)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def providesLocalCache(
|
||||
@Flag(localCacheTtlFlagName) localCacheTtlFlag: Long,
|
||||
@Flag(localCacheMaxSizeFlagName) localCacheMaxSizeFlag: Long,
|
||||
statsReceiver: StatsReceiver
|
||||
): LocalCache[EnrichmentKey, DynMap] = {
|
||||
val underlying = CacheBuilder
|
||||
.newBuilder()
|
||||
.expireAfterWrite(localCacheTtlFlag, TimeUnit.SECONDS)
|
||||
.maximumSize(localCacheMaxSizeFlag)
|
||||
.build[EnrichmentKey, Future[DynMap]]()
|
||||
|
||||
new LocalCache[EnrichmentKey, DynMap](
|
||||
underlying = underlying,
|
||||
statsReceiver = statsReceiver.scope("enricherLocalCache"))
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,24 @@
|
||||
package com.twitter.unified_user_actions.service.module
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.finagle.thrift.ClientId
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.inject.annotations.Flag
|
||||
import javax.inject.Singleton
|
||||
|
||||
object ClientIdModule extends TwitterModule {
|
||||
private final val flagName = "thrift.client.id"
|
||||
|
||||
flag[String](
|
||||
name = flagName,
|
||||
help = "Thrift Client ID"
|
||||
)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def providesClientId(
|
||||
@Flag(flagName) thriftClientId: String,
|
||||
): ClientId = ClientId(
|
||||
name = thriftClientId
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,27 @@
|
||||
package com.twitter.unified_user_actions.service.module
|
||||
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.decider.RandomRecipient
|
||||
import com.twitter.unified_user_actions.thriftscala.ActionType
|
||||
import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction
|
||||
|
||||
sealed trait DeciderUtils {
|
||||
def shouldPublish(decider: Decider, uua: UnifiedUserAction, sinkTopic: String): Boolean
|
||||
}
|
||||
|
||||
object DefaultDeciderUtils extends DeciderUtils {
|
||||
override def shouldPublish(decider: Decider, uua: UnifiedUserAction, sinkTopic: String): Boolean =
|
||||
decider.isAvailable(feature = s"Publish${uua.actionType}", Some(RandomRecipient))
|
||||
}
|
||||
|
||||
object ClientEventDeciderUtils extends DeciderUtils {
|
||||
override def shouldPublish(decider: Decider, uua: UnifiedUserAction, sinkTopic: String): Boolean =
|
||||
decider.isAvailable(
|
||||
feature = s"Publish${uua.actionType}",
|
||||
Some(RandomRecipient)) && (uua.actionType match {
|
||||
// for heavy impressions UUA only publishes to the "all" topic, not the engagementsOnly topic.
|
||||
case ActionType.ClientTweetLingerImpression | ActionType.ClientTweetRenderImpression =>
|
||||
sinkTopic == TopicsMapping().all
|
||||
case _ => true
|
||||
})
|
||||
}
|
||||
@ -0,0 +1,172 @@
|
||||
package com.twitter.unified_user_actions.service.module
|
||||
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.unified_user_actions.kafka.ClientConfigs
|
||||
import com.twitter.unified_user_actions.kafka.CompressionTypeFlag
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.StorageUnit
|
||||
import com.twitter.util.logging.Logging
|
||||
|
||||
object FlagsModule extends TwitterModule with Logging {
|
||||
// Twitter
|
||||
final val cluster = "cluster"
|
||||
|
||||
// Required
|
||||
final val kafkaSourceCluster = ClientConfigs.kafkaBootstrapServerConfig
|
||||
final val kafkaDestCluster = ClientConfigs.kafkaBootstrapServerRemoteDestConfig
|
||||
final val kafkaSourceTopic = "kafka.source.topic"
|
||||
final val kafkaSinkTopics = "kafka.sink.topics"
|
||||
final val kafkaGroupId = ClientConfigs.kafkaGroupIdConfig
|
||||
final val kafkaProducerClientId = ClientConfigs.producerClientIdConfig
|
||||
final val kafkaMaxPendingRequests = ClientConfigs.kafkaMaxPendingRequestsConfig
|
||||
final val kafkaWorkerThreads = ClientConfigs.kafkaWorkerThreadsConfig
|
||||
|
||||
// Optional
|
||||
/// Authentication
|
||||
final val enableTrustStore = ClientConfigs.enableTrustStore
|
||||
final val trustStoreLocation = ClientConfigs.trustStoreLocationConfig
|
||||
|
||||
/// Consumer
|
||||
final val commitInterval = ClientConfigs.kafkaCommitIntervalConfig
|
||||
final val maxPollRecords = ClientConfigs.consumerMaxPollRecordsConfig
|
||||
final val maxPollInterval = ClientConfigs.consumerMaxPollIntervalConfig
|
||||
final val sessionTimeout = ClientConfigs.consumerSessionTimeoutConfig
|
||||
final val fetchMax = ClientConfigs.consumerFetchMaxConfig
|
||||
final val fetchMin = ClientConfigs.consumerFetchMinConfig
|
||||
final val receiveBuffer = ClientConfigs.consumerReceiveBufferSizeConfig
|
||||
/// Producer
|
||||
final val batchSize = ClientConfigs.producerBatchSizeConfig
|
||||
final val linger = ClientConfigs.producerLingerConfig
|
||||
final val bufferMem = ClientConfigs.producerBufferMemConfig
|
||||
final val compressionType = ClientConfigs.compressionConfig
|
||||
final val retries = ClientConfigs.retriesConfig
|
||||
final val retryBackoff = ClientConfigs.retryBackoffConfig
|
||||
final val requestTimeout = ClientConfigs.producerRequestTimeoutConfig
|
||||
|
||||
// Twitter
|
||||
flag[String](
|
||||
name = cluster,
|
||||
help = "The zone (or DC) that this service runs, used to potentially filter events"
|
||||
)
|
||||
|
||||
// Required
|
||||
flag[String](
|
||||
name = kafkaSourceCluster,
|
||||
help = ClientConfigs.kafkaBootstrapServerHelp
|
||||
)
|
||||
flag[String](
|
||||
name = kafkaDestCluster,
|
||||
help = ClientConfigs.kafkaBootstrapServerRemoteDestHelp
|
||||
)
|
||||
flag[String](
|
||||
name = kafkaSourceTopic,
|
||||
help = "Name of the source Kafka topic"
|
||||
)
|
||||
flag[Seq[String]](
|
||||
name = kafkaSinkTopics,
|
||||
help = "A list of sink Kafka topics, separated by comma (,)"
|
||||
)
|
||||
flag[String](
|
||||
name = kafkaGroupId,
|
||||
help = ClientConfigs.kafkaGroupIdHelp
|
||||
)
|
||||
flag[String](
|
||||
name = kafkaProducerClientId,
|
||||
help = ClientConfigs.producerClientIdHelp
|
||||
)
|
||||
flag[Int](
|
||||
name = kafkaMaxPendingRequests,
|
||||
help = ClientConfigs.kafkaMaxPendingRequestsHelp
|
||||
)
|
||||
flag[Int](
|
||||
name = kafkaWorkerThreads,
|
||||
help = ClientConfigs.kafkaWorkerThreadsHelp
|
||||
)
|
||||
|
||||
// Optional
|
||||
/// Authentication
|
||||
flag[Boolean](
|
||||
name = enableTrustStore,
|
||||
default = ClientConfigs.enableTrustStoreDefault,
|
||||
help = ClientConfigs.enableTrustStoreHelp
|
||||
)
|
||||
flag[String](
|
||||
name = trustStoreLocation,
|
||||
default = ClientConfigs.trustStoreLocationDefault,
|
||||
help = ClientConfigs.trustStoreLocationHelp
|
||||
)
|
||||
|
||||
/// Consumer
|
||||
flag[Duration](
|
||||
name = commitInterval,
|
||||
default = ClientConfigs.kafkaCommitIntervalDefault,
|
||||
help = ClientConfigs.kafkaCommitIntervalHelp
|
||||
)
|
||||
flag[Int](
|
||||
name = maxPollRecords,
|
||||
default = ClientConfigs.consumerMaxPollRecordsDefault,
|
||||
help = ClientConfigs.consumerMaxPollRecordsHelp
|
||||
)
|
||||
flag[Duration](
|
||||
name = maxPollInterval,
|
||||
default = ClientConfigs.consumerMaxPollIntervalDefault,
|
||||
help = ClientConfigs.consumerMaxPollIntervalHelp
|
||||
)
|
||||
flag[Duration](
|
||||
name = sessionTimeout,
|
||||
default = ClientConfigs.consumerSessionTimeoutDefault,
|
||||
help = ClientConfigs.consumerSessionTimeoutHelp
|
||||
)
|
||||
flag[StorageUnit](
|
||||
name = fetchMax,
|
||||
default = ClientConfigs.consumerFetchMaxDefault,
|
||||
help = ClientConfigs.consumerFetchMaxHelp
|
||||
)
|
||||
flag[StorageUnit](
|
||||
name = fetchMin,
|
||||
default = ClientConfigs.consumerFetchMinDefault,
|
||||
help = ClientConfigs.consumerFetchMinHelp
|
||||
)
|
||||
flag[StorageUnit](
|
||||
name = receiveBuffer,
|
||||
default = ClientConfigs.consumerReceiveBufferSizeDefault,
|
||||
help = ClientConfigs.consumerReceiveBufferSizeHelp
|
||||
)
|
||||
|
||||
/// Producer
|
||||
flag[StorageUnit](
|
||||
name = batchSize,
|
||||
default = ClientConfigs.producerBatchSizeDefault,
|
||||
help = ClientConfigs.producerBatchSizeHelp
|
||||
)
|
||||
flag[Duration](
|
||||
name = linger,
|
||||
default = ClientConfigs.producerLingerDefault,
|
||||
help = ClientConfigs.producerLingerHelp
|
||||
)
|
||||
flag[StorageUnit](
|
||||
name = bufferMem,
|
||||
default = ClientConfigs.producerBufferMemDefault,
|
||||
help = ClientConfigs.producerBufferMemHelp
|
||||
)
|
||||
flag[CompressionTypeFlag](
|
||||
name = compressionType,
|
||||
default = ClientConfigs.compressionDefault,
|
||||
help = ClientConfigs.compressionHelp
|
||||
)
|
||||
flag[Int](
|
||||
name = retries,
|
||||
default = ClientConfigs.retriesDefault,
|
||||
help = ClientConfigs.retriesHelp
|
||||
)
|
||||
flag[Duration](
|
||||
name = retryBackoff,
|
||||
default = ClientConfigs.retryBackoffDefault,
|
||||
help = ClientConfigs.retryBackoffHelp
|
||||
)
|
||||
flag[Duration](
|
||||
name = requestTimeout,
|
||||
default = ClientConfigs.producerRequestTimeoutDefault,
|
||||
help = ClientConfigs.producerRequestTimeoutHelp
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,42 @@
|
||||
package com.twitter.unified_user_actions.service.module
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.finagle.ThriftMux
|
||||
import com.twitter.finagle.mtls.authentication.ServiceIdentifier
|
||||
import com.twitter.finagle.mtls.client.MtlsStackClient.MtlsThriftMuxClientSyntax
|
||||
import com.twitter.finagle.ssl.OpportunisticTls
|
||||
import com.twitter.finagle.thrift.ClientId
|
||||
import com.twitter.finagle.thrift.RichClientParam
|
||||
import com.twitter.graphql.thriftscala.GraphqlExecutionService
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.util.Duration
|
||||
import javax.inject.Singleton
|
||||
|
||||
object GraphqlClientProviderModule extends TwitterModule {
|
||||
private def buildClient(serviceIdentifier: ServiceIdentifier, clientId: ClientId) =
|
||||
ThriftMux.client
|
||||
.withRequestTimeout(Duration.fromSeconds(5))
|
||||
.withMutualTls(serviceIdentifier)
|
||||
.withOpportunisticTls(OpportunisticTls.Required)
|
||||
.withClientId(clientId)
|
||||
.newService("/s/graphql-service/graphql-api:thrift")
|
||||
|
||||
def buildGraphQlClient(
|
||||
serviceIdentifer: ServiceIdentifier,
|
||||
clientId: ClientId
|
||||
): GraphqlExecutionService.FinagledClient = {
|
||||
val client = buildClient(serviceIdentifer, clientId)
|
||||
new GraphqlExecutionService.FinagledClient(client, RichClientParam())
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def providesGraphQlClient(
|
||||
serviceIdentifier: ServiceIdentifier,
|
||||
clientId: ClientId
|
||||
): GraphqlExecutionService.FinagledClient =
|
||||
buildGraphQlClient(
|
||||
serviceIdentifier,
|
||||
clientId
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,87 @@
|
||||
package com.twitter.unified_user_actions.service.module
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.ads.spendserver.thriftscala.SpendServerEvent
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.finatra.kafka.serde.UnKeyedSerde
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.inject.annotations.Flag
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.unified_user_actions.adapter.ads_callback_engagements.AdsCallbackEngagementsAdapter
|
||||
import com.twitter.unified_user_actions.kafka.CompressionTypeFlag
|
||||
import com.twitter.unified_user_actions.kafka.serde.NullableScalaSerdes
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.StorageUnit
|
||||
import com.twitter.util.logging.Logging
|
||||
import javax.inject.Singleton
|
||||
|
||||
object KafkaProcessorAdsCallbackEngagementsModule extends TwitterModule with Logging {
|
||||
override def modules = Seq(FlagsModule)
|
||||
|
||||
// NOTE: This is a shared processor name in order to simplify monviz stat computation.
|
||||
private final val processorName = "uuaProcessor"
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def providesKafkaProcessor(
|
||||
decider: Decider,
|
||||
@Flag(FlagsModule.cluster) cluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String,
|
||||
@Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String,
|
||||
@Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String],
|
||||
@Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String,
|
||||
@Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String,
|
||||
@Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int,
|
||||
@Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int,
|
||||
@Flag(FlagsModule.commitInterval) commitInterval: Duration,
|
||||
@Flag(FlagsModule.maxPollRecords) maxPollRecords: Int,
|
||||
@Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration,
|
||||
@Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration,
|
||||
@Flag(FlagsModule.fetchMax) fetchMax: StorageUnit,
|
||||
@Flag(FlagsModule.batchSize) batchSize: StorageUnit,
|
||||
@Flag(FlagsModule.linger) linger: Duration,
|
||||
@Flag(FlagsModule.bufferMem) bufferMem: StorageUnit,
|
||||
@Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag,
|
||||
@Flag(FlagsModule.retries) retries: Int,
|
||||
@Flag(FlagsModule.retryBackoff) retryBackoff: Duration,
|
||||
@Flag(FlagsModule.requestTimeout) requestTimeout: Duration,
|
||||
@Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean,
|
||||
@Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String,
|
||||
statsReceiver: StatsReceiver,
|
||||
): AtLeastOnceProcessor[UnKeyed, SpendServerEvent] = {
|
||||
KafkaProcessorProvider.provideDefaultAtLeastOnceProcessor(
|
||||
name = processorName,
|
||||
kafkaSourceCluster = kafkaSourceCluster,
|
||||
kafkaGroupId = kafkaGroupId,
|
||||
kafkaSourceTopic = kafkaSourceTopic,
|
||||
sourceKeyDeserializer = UnKeyedSerde.deserializer,
|
||||
sourceValueDeserializer = NullableScalaSerdes
|
||||
.Thrift[SpendServerEvent](statsReceiver.counter("deserializerErrors")).deserializer,
|
||||
commitInterval = commitInterval,
|
||||
maxPollRecords = maxPollRecords,
|
||||
maxPollInterval = maxPollInterval,
|
||||
sessionTimeout = sessionTimeout,
|
||||
fetchMax = fetchMax,
|
||||
processorMaxPendingRequests = kafkaMaxPendingRequests,
|
||||
processorWorkerThreads = kafkaWorkerThreads,
|
||||
adapter = new AdsCallbackEngagementsAdapter,
|
||||
kafkaSinkTopics = kafkaSinkTopics,
|
||||
kafkaDestCluster = kafkaDestCluster,
|
||||
kafkaProducerClientId = kafkaProducerClientId,
|
||||
batchSize = batchSize,
|
||||
linger = linger,
|
||||
bufferMem = bufferMem,
|
||||
compressionType = compressionTypeFlag.compressionType,
|
||||
retries = retries,
|
||||
retryBackoff = retryBackoff,
|
||||
requestTimeout = requestTimeout,
|
||||
statsReceiver = statsReceiver,
|
||||
trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None,
|
||||
decider = decider,
|
||||
zone = ZoneFiltering.zoneMapping(cluster),
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,87 @@
|
||||
package com.twitter.unified_user_actions.service.module
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.finatra.kafka.serde.UnKeyedSerde
|
||||
import com.twitter.inject.annotations.Flag
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.storage.behavioral_event.thriftscala.FlattenedEventLog
|
||||
import com.twitter.unified_user_actions.adapter.behavioral_client_event.BehavioralClientEventAdapter
|
||||
import com.twitter.unified_user_actions.kafka.CompressionTypeFlag
|
||||
import com.twitter.unified_user_actions.kafka.serde.NullableScalaSerdes
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.StorageUnit
|
||||
import com.twitter.util.logging.Logging
|
||||
import javax.inject.Singleton
|
||||
|
||||
object KafkaProcessorBehavioralClientEventModule extends TwitterModule with Logging {
|
||||
override def modules = Seq(FlagsModule)
|
||||
|
||||
private val adapter: BehavioralClientEventAdapter = new BehavioralClientEventAdapter
|
||||
private final val processorName: String = "uuaProcessor"
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def providesKafkaProcessor(
|
||||
decider: Decider,
|
||||
@Flag(FlagsModule.cluster) cluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String,
|
||||
@Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String,
|
||||
@Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String],
|
||||
@Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String,
|
||||
@Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String,
|
||||
@Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int,
|
||||
@Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int,
|
||||
@Flag(FlagsModule.commitInterval) commitInterval: Duration,
|
||||
@Flag(FlagsModule.maxPollRecords) maxPollRecords: Int,
|
||||
@Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration,
|
||||
@Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration,
|
||||
@Flag(FlagsModule.fetchMax) fetchMax: StorageUnit,
|
||||
@Flag(FlagsModule.batchSize) batchSize: StorageUnit,
|
||||
@Flag(FlagsModule.linger) linger: Duration,
|
||||
@Flag(FlagsModule.bufferMem) bufferMem: StorageUnit,
|
||||
@Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag,
|
||||
@Flag(FlagsModule.retries) retries: Int,
|
||||
@Flag(FlagsModule.retryBackoff) retryBackoff: Duration,
|
||||
@Flag(FlagsModule.requestTimeout) requestTimeout: Duration,
|
||||
@Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean,
|
||||
@Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String,
|
||||
statsReceiver: StatsReceiver,
|
||||
): AtLeastOnceProcessor[UnKeyed, FlattenedEventLog] = {
|
||||
KafkaProcessorProvider.provideDefaultAtLeastOnceProcessor(
|
||||
name = processorName,
|
||||
kafkaSourceCluster = kafkaSourceCluster,
|
||||
kafkaGroupId = kafkaGroupId,
|
||||
kafkaSourceTopic = kafkaSourceTopic,
|
||||
sourceKeyDeserializer = UnKeyedSerde.deserializer,
|
||||
sourceValueDeserializer = NullableScalaSerdes
|
||||
.Thrift[FlattenedEventLog](statsReceiver.counter("deserializerErrors")).deserializer,
|
||||
commitInterval = commitInterval,
|
||||
maxPollRecords = maxPollRecords,
|
||||
maxPollInterval = maxPollInterval,
|
||||
sessionTimeout = sessionTimeout,
|
||||
fetchMax = fetchMax,
|
||||
processorMaxPendingRequests = kafkaMaxPendingRequests,
|
||||
processorWorkerThreads = kafkaWorkerThreads,
|
||||
adapter = adapter,
|
||||
kafkaSinkTopics = kafkaSinkTopics,
|
||||
kafkaDestCluster = kafkaDestCluster,
|
||||
kafkaProducerClientId = kafkaProducerClientId,
|
||||
batchSize = batchSize,
|
||||
linger = linger,
|
||||
bufferMem = bufferMem,
|
||||
compressionType = compressionTypeFlag.compressionType,
|
||||
retries = retries,
|
||||
retryBackoff = retryBackoff,
|
||||
requestTimeout = requestTimeout,
|
||||
statsReceiver = statsReceiver,
|
||||
trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None,
|
||||
decider = decider,
|
||||
zone = ZoneFiltering.zoneMapping(cluster),
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,142 @@
|
||||
package com.twitter.unified_user_actions.service.module
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.clientapp.thriftscala.LogEvent
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finatra.kafka.producers.BlockingFinagleKafkaProducer
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.finatra.kafka.serde.UnKeyedSerde
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.inject.annotations.Flag
|
||||
import com.twitter.kafka.client.headers.Zone
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.unified_user_actions.adapter.client_event.ClientEventAdapter
|
||||
import com.twitter.unified_user_actions.kafka.CompressionTypeFlag
|
||||
import com.twitter.unified_user_actions.kafka.serde.NullableScalaSerdes
|
||||
import com.twitter.unified_user_actions.service.module.KafkaProcessorProvider.updateActionTypeCounters
|
||||
import com.twitter.unified_user_actions.service.module.KafkaProcessorProvider.updateProcessingTimeStats
|
||||
import com.twitter.unified_user_actions.service.module.KafkaProcessorProvider.updateProductSurfaceTypeCounters
|
||||
import com.twitter.unified_user_actions.thriftscala.ActionType
|
||||
import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.Future
|
||||
import com.twitter.util.StorageUnit
|
||||
import com.twitter.util.logging.Logging
|
||||
import javax.inject.Singleton
|
||||
import org.apache.kafka.clients.producer.ProducerRecord
|
||||
import org.apache.kafka.common.header.Headers
|
||||
|
||||
object KafkaProcessorClientEventModule extends TwitterModule with Logging {
|
||||
override def modules: Seq[FlagsModule.type] = Seq(FlagsModule)
|
||||
|
||||
private val clientEventAdapter = new ClientEventAdapter
|
||||
// NOTE: This is a shared processor name in order to simplify monviz stat computation.
|
||||
private final val processorName = "uuaProcessor"
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def providesKafkaProcessor(
|
||||
decider: Decider,
|
||||
@Flag(FlagsModule.cluster) cluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String,
|
||||
@Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String,
|
||||
@Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String],
|
||||
@Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String,
|
||||
@Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String,
|
||||
@Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int,
|
||||
@Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int,
|
||||
@Flag(FlagsModule.commitInterval) commitInterval: Duration,
|
||||
@Flag(FlagsModule.maxPollRecords) maxPollRecords: Int,
|
||||
@Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration,
|
||||
@Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration,
|
||||
@Flag(FlagsModule.fetchMax) fetchMax: StorageUnit,
|
||||
@Flag(FlagsModule.fetchMin) fetchMin: StorageUnit,
|
||||
@Flag(FlagsModule.batchSize) batchSize: StorageUnit,
|
||||
@Flag(FlagsModule.linger) linger: Duration,
|
||||
@Flag(FlagsModule.bufferMem) bufferMem: StorageUnit,
|
||||
@Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag,
|
||||
@Flag(FlagsModule.retries) retries: Int,
|
||||
@Flag(FlagsModule.retryBackoff) retryBackoff: Duration,
|
||||
@Flag(FlagsModule.requestTimeout) requestTimeout: Duration,
|
||||
@Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean,
|
||||
@Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String,
|
||||
statsReceiver: StatsReceiver,
|
||||
): AtLeastOnceProcessor[UnKeyed, LogEvent] = {
|
||||
KafkaProcessorProvider.provideDefaultAtLeastOnceProcessor(
|
||||
name = processorName,
|
||||
kafkaSourceCluster = kafkaSourceCluster,
|
||||
kafkaGroupId = kafkaGroupId,
|
||||
kafkaSourceTopic = kafkaSourceTopic,
|
||||
sourceKeyDeserializer = UnKeyedSerde.deserializer,
|
||||
sourceValueDeserializer = NullableScalaSerdes
|
||||
.Thrift[LogEvent](statsReceiver.counter("deserializerErrors")).deserializer,
|
||||
commitInterval = commitInterval,
|
||||
maxPollRecords = maxPollRecords,
|
||||
maxPollInterval = maxPollInterval,
|
||||
sessionTimeout = sessionTimeout,
|
||||
fetchMax = fetchMax,
|
||||
fetchMin = fetchMin,
|
||||
processorMaxPendingRequests = kafkaMaxPendingRequests,
|
||||
processorWorkerThreads = kafkaWorkerThreads,
|
||||
adapter = clientEventAdapter,
|
||||
kafkaSinkTopics = kafkaSinkTopics,
|
||||
kafkaDestCluster = kafkaDestCluster,
|
||||
kafkaProducerClientId = kafkaProducerClientId,
|
||||
batchSize = batchSize,
|
||||
linger = linger,
|
||||
bufferMem = bufferMem,
|
||||
compressionType = compressionTypeFlag.compressionType,
|
||||
retries = retries,
|
||||
retryBackoff = retryBackoff,
|
||||
requestTimeout = requestTimeout,
|
||||
statsReceiver = statsReceiver,
|
||||
produceOpt = Some(clientEventProducer),
|
||||
trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None,
|
||||
decider = decider,
|
||||
zone = ZoneFiltering.zoneMapping(cluster),
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* ClientEvent producer is different from the defaultProducer.
|
||||
* While the defaultProducer publishes every event to all sink topics, ClientEventProducer (this producer) requires
|
||||
* exactly 2 sink topics: Topic with all events (impressions and engagements) and Topic with engagements only.
|
||||
* And the publishing is based the action type.
|
||||
*/
|
||||
def clientEventProducer(
|
||||
producer: BlockingFinagleKafkaProducer[UnKeyed, UnifiedUserAction],
|
||||
k: UnKeyed,
|
||||
v: UnifiedUserAction,
|
||||
sinkTopic: String,
|
||||
headers: Headers,
|
||||
statsReceiver: StatsReceiver,
|
||||
decider: Decider
|
||||
): Future[Unit] =
|
||||
if (ClientEventDeciderUtils.shouldPublish(decider = decider, uua = v, sinkTopic = sinkTopic)) {
|
||||
updateActionTypeCounters(statsReceiver, v, sinkTopic)
|
||||
updateProductSurfaceTypeCounters(statsReceiver, v, sinkTopic)
|
||||
updateProcessingTimeStats(statsReceiver, v)
|
||||
|
||||
// If we were to enable xDC replicator, then we can safely remove the Zone header since xDC
|
||||
// replicator works in the following way:
|
||||
// - If the message does not have a header, the replicator will assume it is local and
|
||||
// set the header, copy the message
|
||||
// - If the message has a header that is the local zone, the replicator will copy the message
|
||||
// - If the message has a header for a different zone, the replicator will drop the message
|
||||
producer
|
||||
.send(
|
||||
new ProducerRecord[UnKeyed, UnifiedUserAction](
|
||||
sinkTopic,
|
||||
null,
|
||||
k,
|
||||
v,
|
||||
headers.remove(Zone.Key)))
|
||||
.onSuccess { _ => statsReceiver.counter("publishSuccess", sinkTopic).incr() }
|
||||
.onFailure { e: Throwable =>
|
||||
statsReceiver.counter("publishFailure", sinkTopic).incr()
|
||||
error(s"Publish error to topic $sinkTopic: $e")
|
||||
}.unit
|
||||
} else Future.Unit
|
||||
}
|
||||
@ -0,0 +1,88 @@
|
||||
package com.twitter.unified_user_actions.service.module
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.finatra.kafka.serde.UnKeyedSerde
|
||||
import com.twitter.ibis.thriftscala.NotificationScribe
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.inject.annotations.Flag
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.unified_user_actions.kafka.CompressionTypeFlag
|
||||
import com.twitter.unified_user_actions.kafka.serde.NullableScalaSerdes
|
||||
import com.twitter.unified_user_actions.adapter.email_notification_event.EmailNotificationEventAdapter
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.StorageUnit
|
||||
import com.twitter.util.logging.Logging
|
||||
import javax.inject.Singleton
|
||||
|
||||
object KafkaProcessorEmailNotificationEventModule extends TwitterModule with Logging {
|
||||
override def modules = Seq(FlagsModule)
|
||||
private val notificationEventAdapter = new EmailNotificationEventAdapter
|
||||
// NOTE: This is a shared processor name in order to simplify monviz stat computation.
|
||||
private final val processorName = "uuaProcessor"
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def providesKafkaProcessor(
|
||||
decider: Decider,
|
||||
@Flag(FlagsModule.cluster) cluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String,
|
||||
@Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String,
|
||||
@Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String],
|
||||
@Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String,
|
||||
@Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String,
|
||||
@Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int,
|
||||
@Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int,
|
||||
@Flag(FlagsModule.commitInterval) commitInterval: Duration,
|
||||
@Flag(FlagsModule.maxPollRecords) maxPollRecords: Int,
|
||||
@Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration,
|
||||
@Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration,
|
||||
@Flag(FlagsModule.fetchMax) fetchMax: StorageUnit,
|
||||
@Flag(FlagsModule.batchSize) batchSize: StorageUnit,
|
||||
@Flag(FlagsModule.linger) linger: Duration,
|
||||
@Flag(FlagsModule.bufferMem) bufferMem: StorageUnit,
|
||||
@Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag,
|
||||
@Flag(FlagsModule.retries) retries: Int,
|
||||
@Flag(FlagsModule.retryBackoff) retryBackoff: Duration,
|
||||
@Flag(FlagsModule.requestTimeout) requestTimeout: Duration,
|
||||
@Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean,
|
||||
@Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String,
|
||||
statsReceiver: StatsReceiver,
|
||||
): AtLeastOnceProcessor[UnKeyed, NotificationScribe] = {
|
||||
KafkaProcessorProvider.provideDefaultAtLeastOnceProcessor(
|
||||
name = processorName,
|
||||
kafkaSourceCluster = kafkaSourceCluster,
|
||||
kafkaGroupId = kafkaGroupId,
|
||||
kafkaSourceTopic = kafkaSourceTopic,
|
||||
sourceKeyDeserializer = UnKeyedSerde.deserializer,
|
||||
sourceValueDeserializer = NullableScalaSerdes
|
||||
.Thrift[NotificationScribe](statsReceiver.counter("deserializerErrors")).deserializer,
|
||||
commitInterval = commitInterval,
|
||||
maxPollRecords = maxPollRecords,
|
||||
maxPollInterval = maxPollInterval,
|
||||
sessionTimeout = sessionTimeout,
|
||||
fetchMax = fetchMax,
|
||||
processorMaxPendingRequests = kafkaMaxPendingRequests,
|
||||
processorWorkerThreads = kafkaWorkerThreads,
|
||||
adapter = notificationEventAdapter,
|
||||
kafkaSinkTopics = kafkaSinkTopics,
|
||||
kafkaDestCluster = kafkaDestCluster,
|
||||
kafkaProducerClientId = kafkaProducerClientId,
|
||||
batchSize = batchSize,
|
||||
linger = linger,
|
||||
bufferMem = bufferMem,
|
||||
compressionType = compressionTypeFlag.compressionType,
|
||||
retries = retries,
|
||||
retryBackoff = retryBackoff,
|
||||
requestTimeout = requestTimeout,
|
||||
statsReceiver = statsReceiver,
|
||||
trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None,
|
||||
decider = decider,
|
||||
zone = ZoneFiltering.zoneMapping(cluster),
|
||||
maybeProcess = ZoneFiltering.localDCFiltering
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,88 @@
|
||||
package com.twitter.unified_user_actions.service.module
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.finatra.kafka.serde.UnKeyedSerde
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.inject.annotations.Flag
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.unified_user_actions.adapter.favorite_archival_events.FavoriteArchivalEventsAdapter
|
||||
import com.twitter.unified_user_actions.kafka.CompressionTypeFlag
|
||||
import com.twitter.unified_user_actions.kafka.serde.NullableScalaSerdes
|
||||
import com.twitter.timelineservice.fanout.thriftscala.FavoriteArchivalEvent
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.StorageUnit
|
||||
import com.twitter.util.logging.Logging
|
||||
import javax.inject.Singleton
|
||||
|
||||
object KafkaProcessorFavoriteArchivalEventsModule extends TwitterModule with Logging {
|
||||
override def modules = Seq(FlagsModule)
|
||||
|
||||
private val adapter = new FavoriteArchivalEventsAdapter
|
||||
// NOTE: This is a shared processor name in order to simplify monviz stat computation.
|
||||
private final val processorName = "uuaProcessor"
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def providesKafkaProcessor(
|
||||
decider: Decider,
|
||||
@Flag(FlagsModule.cluster) cluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String,
|
||||
@Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String,
|
||||
@Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String],
|
||||
@Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String,
|
||||
@Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String,
|
||||
@Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int,
|
||||
@Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int,
|
||||
@Flag(FlagsModule.commitInterval) commitInterval: Duration,
|
||||
@Flag(FlagsModule.maxPollRecords) maxPollRecords: Int,
|
||||
@Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration,
|
||||
@Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration,
|
||||
@Flag(FlagsModule.fetchMax) fetchMax: StorageUnit,
|
||||
@Flag(FlagsModule.batchSize) batchSize: StorageUnit,
|
||||
@Flag(FlagsModule.linger) linger: Duration,
|
||||
@Flag(FlagsModule.bufferMem) bufferMem: StorageUnit,
|
||||
@Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag,
|
||||
@Flag(FlagsModule.retries) retries: Int,
|
||||
@Flag(FlagsModule.retryBackoff) retryBackoff: Duration,
|
||||
@Flag(FlagsModule.requestTimeout) requestTimeout: Duration,
|
||||
@Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean,
|
||||
@Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String,
|
||||
statsReceiver: StatsReceiver,
|
||||
): AtLeastOnceProcessor[UnKeyed, FavoriteArchivalEvent] = {
|
||||
KafkaProcessorProvider.provideDefaultAtLeastOnceProcessor(
|
||||
name = processorName,
|
||||
kafkaSourceCluster = kafkaSourceCluster,
|
||||
kafkaGroupId = kafkaGroupId,
|
||||
kafkaSourceTopic = kafkaSourceTopic,
|
||||
sourceKeyDeserializer = UnKeyedSerde.deserializer,
|
||||
sourceValueDeserializer = NullableScalaSerdes
|
||||
.Thrift[FavoriteArchivalEvent](statsReceiver.counter("deserializerErrors")).deserializer,
|
||||
commitInterval = commitInterval,
|
||||
maxPollRecords = maxPollRecords,
|
||||
maxPollInterval = maxPollInterval,
|
||||
sessionTimeout = sessionTimeout,
|
||||
fetchMax = fetchMax,
|
||||
processorMaxPendingRequests = kafkaMaxPendingRequests,
|
||||
processorWorkerThreads = kafkaWorkerThreads,
|
||||
adapter = adapter,
|
||||
kafkaSinkTopics = kafkaSinkTopics,
|
||||
kafkaDestCluster = kafkaDestCluster,
|
||||
kafkaProducerClientId = kafkaProducerClientId,
|
||||
batchSize = batchSize,
|
||||
linger = linger,
|
||||
bufferMem = bufferMem,
|
||||
compressionType = compressionTypeFlag.compressionType,
|
||||
retries = retries,
|
||||
retryBackoff = retryBackoff,
|
||||
requestTimeout = requestTimeout,
|
||||
statsReceiver = statsReceiver,
|
||||
trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None,
|
||||
decider = decider,
|
||||
zone = ZoneFiltering.zoneMapping(cluster),
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,271 @@
|
||||
package com.twitter.unified_user_actions.service.module
|
||||
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.finagle.stats.Counter
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finatra.kafka.producers.BlockingFinagleKafkaProducer
|
||||
import com.twitter.finatra.kafka.serde.ScalaSerdes
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.finatra.kafka.serde.UnKeyedSerde
|
||||
import com.twitter.kafka.client.headers.Implicits._
|
||||
import com.twitter.kafka.client.headers.Zone
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.unified_user_actions.adapter.AbstractAdapter
|
||||
import com.twitter.unified_user_actions.kafka.ClientConfigs
|
||||
import com.twitter.unified_user_actions.kafka.ClientProviders
|
||||
import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.Future
|
||||
import com.twitter.util.StorageUnit
|
||||
import com.twitter.util.logging.Logging
|
||||
import org.apache.kafka.clients.consumer.ConsumerRecord
|
||||
import org.apache.kafka.clients.producer.ProducerRecord
|
||||
import org.apache.kafka.common.header.Headers
|
||||
import org.apache.kafka.common.record.CompressionType
|
||||
import org.apache.kafka.common.serialization.Deserializer
|
||||
|
||||
object KafkaProcessorProvider extends Logging {
|
||||
lazy val actionTypeStatsCounterMap: collection.mutable.Map[String, Counter] =
|
||||
collection.mutable.Map.empty
|
||||
lazy val productSurfaceTypeStatsCounterMap: collection.mutable.Map[String, Counter] =
|
||||
collection.mutable.Map.empty
|
||||
|
||||
def updateActionTypeCounters(
|
||||
statsReceiver: StatsReceiver,
|
||||
v: UnifiedUserAction,
|
||||
topic: String
|
||||
): Unit = {
|
||||
val actionType = v.actionType.name
|
||||
val actionTypeAndTopicKey = s"$actionType-$topic"
|
||||
actionTypeStatsCounterMap.get(actionTypeAndTopicKey) match {
|
||||
case Some(actionCounter) => actionCounter.incr()
|
||||
case _ =>
|
||||
actionTypeStatsCounterMap(actionTypeAndTopicKey) =
|
||||
statsReceiver.counter("uuaActionType", topic, actionType)
|
||||
actionTypeStatsCounterMap(actionTypeAndTopicKey).incr()
|
||||
}
|
||||
}
|
||||
|
||||
def updateProductSurfaceTypeCounters(
|
||||
statsReceiver: StatsReceiver,
|
||||
v: UnifiedUserAction,
|
||||
topic: String
|
||||
): Unit = {
|
||||
val productSurfaceType = v.productSurface.map(_.name).getOrElse("null")
|
||||
val productSurfaceTypeAndTopicKey = s"$productSurfaceType-$topic"
|
||||
productSurfaceTypeStatsCounterMap.get(productSurfaceTypeAndTopicKey) match {
|
||||
case Some(productSurfaceCounter) => productSurfaceCounter.incr()
|
||||
case _ =>
|
||||
productSurfaceTypeStatsCounterMap(productSurfaceTypeAndTopicKey) =
|
||||
statsReceiver.counter("uuaProductSurfaceType", topic, productSurfaceType)
|
||||
productSurfaceTypeStatsCounterMap(productSurfaceTypeAndTopicKey).incr()
|
||||
}
|
||||
}
|
||||
|
||||
def updateProcessingTimeStats(statsReceiver: StatsReceiver, v: UnifiedUserAction): Unit = {
|
||||
statsReceiver
|
||||
.stat("uuaProcessingTimeDiff").add(
|
||||
v.eventMetadata.receivedTimestampMs - v.eventMetadata.sourceTimestampMs)
|
||||
}
|
||||
|
||||
def defaultProducer(
|
||||
producer: BlockingFinagleKafkaProducer[UnKeyed, UnifiedUserAction],
|
||||
k: UnKeyed,
|
||||
v: UnifiedUserAction,
|
||||
sinkTopic: String,
|
||||
headers: Headers,
|
||||
statsReceiver: StatsReceiver,
|
||||
decider: Decider,
|
||||
): Future[Unit] =
|
||||
if (DefaultDeciderUtils.shouldPublish(decider = decider, uua = v, sinkTopic = sinkTopic)) {
|
||||
updateActionTypeCounters(statsReceiver, v, sinkTopic)
|
||||
updateProcessingTimeStats(statsReceiver, v)
|
||||
|
||||
// If we were to enable xDC replicator, then we can safely remove the Zone header since xDC
|
||||
// replicator works in the following way:
|
||||
// - If the message does not have a header, the replicator will assume it is local and
|
||||
// set the header, copy the message
|
||||
// - If the message has a header that is the local zone, the replicator will copy the message
|
||||
// - If the message has a header for a different zone, the replicator will drop the message
|
||||
producer
|
||||
.send(
|
||||
new ProducerRecord[UnKeyed, UnifiedUserAction](
|
||||
sinkTopic,
|
||||
null,
|
||||
k,
|
||||
v,
|
||||
headers.remove(Zone.Key)))
|
||||
.onSuccess { _ => statsReceiver.counter("publishSuccess", sinkTopic).incr() }
|
||||
.onFailure { e: Throwable =>
|
||||
statsReceiver.counter("publishFailure", sinkTopic).incr()
|
||||
error(s"Publish error to topic $sinkTopic: $e")
|
||||
}.unit
|
||||
} else Future.Unit
|
||||
|
||||
/**
|
||||
* The default AtLeastOnceProcessor mainly for consuming from a single Kafka topic -> process/adapt -> publish to
|
||||
* the single sink Kafka topic.
|
||||
*
|
||||
* Important Note: Currently all sink topics share the same Kafka producer!!! If you need to create different
|
||||
* producers for different topics, you would need to create a customized function like this one.
|
||||
*/
|
||||
def provideDefaultAtLeastOnceProcessor[K, V](
|
||||
name: String,
|
||||
kafkaSourceCluster: String,
|
||||
kafkaGroupId: String,
|
||||
kafkaSourceTopic: String,
|
||||
sourceKeyDeserializer: Deserializer[K],
|
||||
sourceValueDeserializer: Deserializer[V],
|
||||
commitInterval: Duration = ClientConfigs.kafkaCommitIntervalDefault,
|
||||
maxPollRecords: Int = ClientConfigs.consumerMaxPollRecordsDefault,
|
||||
maxPollInterval: Duration = ClientConfigs.consumerMaxPollIntervalDefault,
|
||||
sessionTimeout: Duration = ClientConfigs.consumerSessionTimeoutDefault,
|
||||
fetchMax: StorageUnit = ClientConfigs.consumerFetchMaxDefault,
|
||||
fetchMin: StorageUnit = ClientConfigs.consumerFetchMinDefault,
|
||||
receiveBuffer: StorageUnit = ClientConfigs.consumerReceiveBufferSizeDefault,
|
||||
processorMaxPendingRequests: Int,
|
||||
processorWorkerThreads: Int,
|
||||
adapter: AbstractAdapter[V, UnKeyed, UnifiedUserAction],
|
||||
kafkaSinkTopics: Seq[String],
|
||||
kafkaDestCluster: String,
|
||||
kafkaProducerClientId: String,
|
||||
batchSize: StorageUnit = ClientConfigs.producerBatchSizeDefault,
|
||||
linger: Duration = ClientConfigs.producerLingerDefault,
|
||||
bufferMem: StorageUnit = ClientConfigs.producerBufferMemDefault,
|
||||
compressionType: CompressionType = ClientConfigs.compressionDefault.compressionType,
|
||||
retries: Int = ClientConfigs.retriesDefault,
|
||||
retryBackoff: Duration = ClientConfigs.retryBackoffDefault,
|
||||
requestTimeout: Duration = ClientConfigs.producerRequestTimeoutDefault,
|
||||
produceOpt: Option[
|
||||
(BlockingFinagleKafkaProducer[UnKeyed, UnifiedUserAction], UnKeyed, UnifiedUserAction, String,
|
||||
Headers, StatsReceiver, Decider) => Future[Unit]
|
||||
] = None,
|
||||
trustStoreLocationOpt: Option[String] = Some(ClientConfigs.trustStoreLocationDefault),
|
||||
statsReceiver: StatsReceiver,
|
||||
decider: Decider,
|
||||
zone: Zone,
|
||||
maybeProcess: (ConsumerRecord[K, V], Zone) => Boolean = ZoneFiltering.localDCFiltering[K, V] _,
|
||||
): AtLeastOnceProcessor[K, V] = {
|
||||
|
||||
lazy val singletonProducer = ClientProviders.mkProducer[UnKeyed, UnifiedUserAction](
|
||||
bootstrapServer = kafkaDestCluster,
|
||||
clientId = kafkaProducerClientId,
|
||||
keySerde = UnKeyedSerde.serializer,
|
||||
valueSerde = ScalaSerdes.Thrift[UnifiedUserAction].serializer,
|
||||
idempotence = false,
|
||||
batchSize = batchSize,
|
||||
linger = linger,
|
||||
bufferMem = bufferMem,
|
||||
compressionType = compressionType,
|
||||
retries = retries,
|
||||
retryBackoff = retryBackoff,
|
||||
requestTimeout = requestTimeout,
|
||||
trustStoreLocationOpt = trustStoreLocationOpt,
|
||||
)
|
||||
|
||||
mkAtLeastOnceProcessor[K, V, UnKeyed, UnifiedUserAction](
|
||||
name = name,
|
||||
kafkaSourceCluster = kafkaSourceCluster,
|
||||
kafkaGroupId = kafkaGroupId,
|
||||
kafkaSourceTopic = kafkaSourceTopic,
|
||||
sourceKeyDeserializer = sourceKeyDeserializer,
|
||||
sourceValueDeserializer = sourceValueDeserializer,
|
||||
commitInterval = commitInterval,
|
||||
maxPollRecords = maxPollRecords,
|
||||
maxPollInterval = maxPollInterval,
|
||||
sessionTimeout = sessionTimeout,
|
||||
fetchMax = fetchMax,
|
||||
fetchMin = fetchMin,
|
||||
receiveBuffer = receiveBuffer,
|
||||
processorMaxPendingRequests = processorMaxPendingRequests,
|
||||
processorWorkerThreads = processorWorkerThreads,
|
||||
adapter = adapter,
|
||||
kafkaProducersAndSinkTopics =
|
||||
kafkaSinkTopics.map(sinkTopic => (singletonProducer, sinkTopic)),
|
||||
produce = produceOpt.getOrElse(defaultProducer),
|
||||
trustStoreLocationOpt = trustStoreLocationOpt,
|
||||
statsReceiver = statsReceiver,
|
||||
decider = decider,
|
||||
zone = zone,
|
||||
maybeProcess = maybeProcess,
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* A common AtLeastOnceProcessor provider
|
||||
*/
|
||||
def mkAtLeastOnceProcessor[K, V, OUTK, OUTV](
|
||||
name: String,
|
||||
kafkaSourceCluster: String,
|
||||
kafkaGroupId: String,
|
||||
kafkaSourceTopic: String,
|
||||
sourceKeyDeserializer: Deserializer[K],
|
||||
sourceValueDeserializer: Deserializer[V],
|
||||
commitInterval: Duration = ClientConfigs.kafkaCommitIntervalDefault,
|
||||
maxPollRecords: Int = ClientConfigs.consumerMaxPollRecordsDefault,
|
||||
maxPollInterval: Duration = ClientConfigs.consumerMaxPollIntervalDefault,
|
||||
sessionTimeout: Duration = ClientConfigs.consumerSessionTimeoutDefault,
|
||||
fetchMax: StorageUnit = ClientConfigs.consumerFetchMaxDefault,
|
||||
fetchMin: StorageUnit = ClientConfigs.consumerFetchMinDefault,
|
||||
receiveBuffer: StorageUnit = ClientConfigs.consumerReceiveBufferSizeDefault,
|
||||
processorMaxPendingRequests: Int,
|
||||
processorWorkerThreads: Int,
|
||||
adapter: AbstractAdapter[V, OUTK, OUTV],
|
||||
kafkaProducersAndSinkTopics: Seq[(BlockingFinagleKafkaProducer[OUTK, OUTV], String)],
|
||||
produce: (BlockingFinagleKafkaProducer[OUTK, OUTV], OUTK, OUTV, String, Headers, StatsReceiver,
|
||||
Decider) => Future[Unit],
|
||||
trustStoreLocationOpt: Option[String] = Some(ClientConfigs.trustStoreLocationDefault),
|
||||
statsReceiver: StatsReceiver,
|
||||
decider: Decider,
|
||||
zone: Zone,
|
||||
maybeProcess: (ConsumerRecord[K, V], Zone) => Boolean = ZoneFiltering.localDCFiltering[K, V] _,
|
||||
): AtLeastOnceProcessor[K, V] = {
|
||||
val threadSafeKafkaClient =
|
||||
ClientProviders.mkConsumer[K, V](
|
||||
bootstrapServer = kafkaSourceCluster,
|
||||
keySerde = sourceKeyDeserializer,
|
||||
valueSerde = sourceValueDeserializer,
|
||||
groupId = kafkaGroupId,
|
||||
autoCommit = false,
|
||||
maxPollRecords = maxPollRecords,
|
||||
maxPollInterval = maxPollInterval,
|
||||
sessionTimeout = sessionTimeout,
|
||||
fetchMax = fetchMax,
|
||||
fetchMin = fetchMin,
|
||||
receiveBuffer = receiveBuffer,
|
||||
trustStoreLocationOpt = trustStoreLocationOpt
|
||||
)
|
||||
|
||||
def publish(
|
||||
event: ConsumerRecord[K, V]
|
||||
): Future[Unit] = {
|
||||
statsReceiver.counter("consumedEvents").incr()
|
||||
|
||||
if (maybeProcess(event, zone))
|
||||
Future
|
||||
.collect(
|
||||
adapter
|
||||
.adaptOneToKeyedMany(event.value, statsReceiver)
|
||||
.flatMap {
|
||||
case (k, v) =>
|
||||
kafkaProducersAndSinkTopics.map {
|
||||
case (producer, sinkTopic) =>
|
||||
produce(producer, k, v, sinkTopic, event.headers(), statsReceiver, decider)
|
||||
}
|
||||
}).unit
|
||||
else
|
||||
Future.Unit
|
||||
}
|
||||
|
||||
AtLeastOnceProcessor[K, V](
|
||||
name = name,
|
||||
topic = kafkaSourceTopic,
|
||||
consumer = threadSafeKafkaClient,
|
||||
processor = publish,
|
||||
maxPendingRequests = processorMaxPendingRequests,
|
||||
workerThreads = processorWorkerThreads,
|
||||
commitIntervalMs = commitInterval.inMilliseconds,
|
||||
statsReceiver = statsReceiver
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,207 @@
|
||||
package com.twitter.unified_user_actions.service.module
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.decider.SimpleRecipient
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finatra.kafka.producers.BlockingFinagleKafkaProducer
|
||||
import com.twitter.finatra.kafka.serde.ScalaSerdes
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.finatra.kafka.serde.UnKeyedSerde
|
||||
import com.twitter.iesource.thriftscala.InteractionEvent
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.inject.annotations.Flag
|
||||
import com.twitter.kafka.client.headers.Zone
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.unified_user_actions.adapter.AbstractAdapter
|
||||
import com.twitter.unified_user_actions.adapter.uua_aggregates.RekeyUuaFromInteractionEventsAdapter
|
||||
import com.twitter.unified_user_actions.kafka.ClientConfigs
|
||||
import com.twitter.unified_user_actions.kafka.ClientProviders
|
||||
import com.twitter.unified_user_actions.kafka.CompressionTypeFlag
|
||||
import com.twitter.unified_user_actions.thriftscala.KeyedUuaTweet
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.Future
|
||||
import com.twitter.util.StorageUnit
|
||||
import com.twitter.util.logging.Logging
|
||||
import org.apache.kafka.clients.consumer.ConsumerRecord
|
||||
import org.apache.kafka.clients.producer.ProducerRecord
|
||||
import org.apache.kafka.common.header.Headers
|
||||
import org.apache.kafka.common.record.CompressionType
|
||||
import javax.inject.Singleton
|
||||
import javax.inject.Inject
|
||||
|
||||
object KafkaProcessorRekeyUuaIesourceModule extends TwitterModule with Logging {
|
||||
override def modules = Seq(FlagsModule)
|
||||
|
||||
private val adapter = new RekeyUuaFromInteractionEventsAdapter
|
||||
// NOTE: This is a shared processor name in order to simplify monviz stat computation.
|
||||
private final val processorName = "uuaProcessor"
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Inject
|
||||
def providesKafkaProcessor(
|
||||
decider: Decider,
|
||||
@Flag(FlagsModule.cluster) cluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String,
|
||||
@Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String,
|
||||
@Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String],
|
||||
@Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String,
|
||||
@Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String,
|
||||
@Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int,
|
||||
@Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int,
|
||||
@Flag(FlagsModule.commitInterval) commitInterval: Duration,
|
||||
@Flag(FlagsModule.maxPollRecords) maxPollRecords: Int,
|
||||
@Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration,
|
||||
@Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration,
|
||||
@Flag(FlagsModule.fetchMax) fetchMax: StorageUnit,
|
||||
@Flag(FlagsModule.receiveBuffer) receiveBuffer: StorageUnit,
|
||||
@Flag(FlagsModule.batchSize) batchSize: StorageUnit,
|
||||
@Flag(FlagsModule.linger) linger: Duration,
|
||||
@Flag(FlagsModule.bufferMem) bufferMem: StorageUnit,
|
||||
@Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag,
|
||||
@Flag(FlagsModule.retries) retries: Int,
|
||||
@Flag(FlagsModule.retryBackoff) retryBackoff: Duration,
|
||||
@Flag(FlagsModule.requestTimeout) requestTimeout: Duration,
|
||||
@Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean,
|
||||
@Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String,
|
||||
statsReceiver: StatsReceiver,
|
||||
): AtLeastOnceProcessor[UnKeyed, InteractionEvent] = {
|
||||
provideAtLeastOnceProcessor(
|
||||
name = processorName,
|
||||
kafkaSourceCluster = kafkaSourceCluster,
|
||||
kafkaGroupId = kafkaGroupId,
|
||||
kafkaSourceTopic = kafkaSourceTopic,
|
||||
commitInterval = commitInterval,
|
||||
maxPollRecords = maxPollRecords,
|
||||
maxPollInterval = maxPollInterval,
|
||||
sessionTimeout = sessionTimeout,
|
||||
fetchMax = fetchMax,
|
||||
receiveBuffer = receiveBuffer,
|
||||
processorMaxPendingRequests = kafkaMaxPendingRequests,
|
||||
processorWorkerThreads = kafkaWorkerThreads,
|
||||
adapter = adapter,
|
||||
kafkaSinkTopics = kafkaSinkTopics,
|
||||
kafkaDestCluster = kafkaDestCluster,
|
||||
kafkaProducerClientId = kafkaProducerClientId,
|
||||
batchSize = batchSize,
|
||||
linger = linger,
|
||||
bufferMem = bufferMem,
|
||||
compressionType = compressionTypeFlag.compressionType,
|
||||
retries = retries,
|
||||
retryBackoff = retryBackoff,
|
||||
requestTimeout = requestTimeout,
|
||||
statsReceiver = statsReceiver,
|
||||
trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None,
|
||||
decider = decider,
|
||||
zone = ZoneFiltering.zoneMapping(cluster),
|
||||
maybeProcess = ZoneFiltering.noFiltering
|
||||
)
|
||||
}
|
||||
|
||||
def producer(
|
||||
producer: BlockingFinagleKafkaProducer[Long, KeyedUuaTweet],
|
||||
k: Long,
|
||||
v: KeyedUuaTweet,
|
||||
sinkTopic: String,
|
||||
headers: Headers,
|
||||
statsReceiver: StatsReceiver,
|
||||
decider: Decider,
|
||||
): Future[Unit] =
|
||||
if (decider.isAvailable(feature = s"RekeyUUAIesource${v.actionType}", Some(SimpleRecipient(k))))
|
||||
// If we were to enable xDC replicator, then we can safely remove the Zone header since xDC
|
||||
// replicator works in the following way:
|
||||
// - If the message does not have a header, the replicator will assume it is local and
|
||||
// set the header, copy the message
|
||||
// - If the message has a header that is the local zone, the replicator will copy the message
|
||||
// - If the message has a header for a different zone, the replicator will drop the message
|
||||
producer
|
||||
.send(new ProducerRecord[Long, KeyedUuaTweet](sinkTopic, null, k, v, headers))
|
||||
.onSuccess { _ => statsReceiver.counter("publishSuccess", sinkTopic).incr() }
|
||||
.onFailure { e: Throwable =>
|
||||
statsReceiver.counter("publishFailure", sinkTopic).incr()
|
||||
error(s"Publish error to topic $sinkTopic: $e")
|
||||
}.unit
|
||||
else Future.Unit
|
||||
|
||||
def provideAtLeastOnceProcessor(
|
||||
name: String,
|
||||
kafkaSourceCluster: String,
|
||||
kafkaGroupId: String,
|
||||
kafkaSourceTopic: String,
|
||||
commitInterval: Duration = ClientConfigs.kafkaCommitIntervalDefault,
|
||||
maxPollRecords: Int = ClientConfigs.consumerMaxPollRecordsDefault,
|
||||
maxPollInterval: Duration = ClientConfigs.consumerMaxPollIntervalDefault,
|
||||
sessionTimeout: Duration = ClientConfigs.consumerSessionTimeoutDefault,
|
||||
fetchMax: StorageUnit = ClientConfigs.consumerFetchMaxDefault,
|
||||
fetchMin: StorageUnit = ClientConfigs.consumerFetchMinDefault,
|
||||
receiveBuffer: StorageUnit = ClientConfigs.consumerReceiveBufferSizeDefault,
|
||||
processorMaxPendingRequests: Int,
|
||||
processorWorkerThreads: Int,
|
||||
adapter: AbstractAdapter[InteractionEvent, Long, KeyedUuaTweet],
|
||||
kafkaSinkTopics: Seq[String],
|
||||
kafkaDestCluster: String,
|
||||
kafkaProducerClientId: String,
|
||||
batchSize: StorageUnit = ClientConfigs.producerBatchSizeDefault,
|
||||
linger: Duration = ClientConfigs.producerLingerDefault,
|
||||
bufferMem: StorageUnit = ClientConfigs.producerBufferMemDefault,
|
||||
compressionType: CompressionType = ClientConfigs.compressionDefault.compressionType,
|
||||
retries: Int = ClientConfigs.retriesDefault,
|
||||
retryBackoff: Duration = ClientConfigs.retryBackoffDefault,
|
||||
requestTimeout: Duration = ClientConfigs.producerRequestTimeoutDefault,
|
||||
produceOpt: Option[
|
||||
(BlockingFinagleKafkaProducer[Long, KeyedUuaTweet], Long, KeyedUuaTweet, String, Headers,
|
||||
StatsReceiver, Decider) => Future[Unit]
|
||||
] = Some(producer),
|
||||
trustStoreLocationOpt: Option[String] = Some(ClientConfigs.trustStoreLocationDefault),
|
||||
statsReceiver: StatsReceiver,
|
||||
decider: Decider,
|
||||
zone: Zone,
|
||||
maybeProcess: (ConsumerRecord[UnKeyed, InteractionEvent], Zone) => Boolean,
|
||||
): AtLeastOnceProcessor[UnKeyed, InteractionEvent] = {
|
||||
|
||||
lazy val singletonProducer = ClientProviders.mkProducer[Long, KeyedUuaTweet](
|
||||
bootstrapServer = kafkaDestCluster,
|
||||
clientId = kafkaProducerClientId,
|
||||
keySerde = ScalaSerdes.Long.serializer,
|
||||
valueSerde = ScalaSerdes.Thrift[KeyedUuaTweet].serializer,
|
||||
idempotence = false,
|
||||
batchSize = batchSize,
|
||||
linger = linger,
|
||||
bufferMem = bufferMem,
|
||||
compressionType = compressionType,
|
||||
retries = retries,
|
||||
retryBackoff = retryBackoff,
|
||||
requestTimeout = requestTimeout,
|
||||
trustStoreLocationOpt = trustStoreLocationOpt,
|
||||
)
|
||||
|
||||
KafkaProcessorProvider.mkAtLeastOnceProcessor[UnKeyed, InteractionEvent, Long, KeyedUuaTweet](
|
||||
name = name,
|
||||
kafkaSourceCluster = kafkaSourceCluster,
|
||||
kafkaGroupId = kafkaGroupId,
|
||||
kafkaSourceTopic = kafkaSourceTopic,
|
||||
sourceKeyDeserializer = UnKeyedSerde.deserializer,
|
||||
sourceValueDeserializer = ScalaSerdes.CompactThrift[InteractionEvent].deserializer,
|
||||
commitInterval = commitInterval,
|
||||
maxPollRecords = maxPollRecords,
|
||||
maxPollInterval = maxPollInterval,
|
||||
sessionTimeout = sessionTimeout,
|
||||
fetchMax = fetchMax,
|
||||
fetchMin = fetchMin,
|
||||
receiveBuffer = receiveBuffer,
|
||||
processorMaxPendingRequests = processorMaxPendingRequests,
|
||||
processorWorkerThreads = processorWorkerThreads,
|
||||
adapter = adapter,
|
||||
kafkaProducersAndSinkTopics =
|
||||
kafkaSinkTopics.map(sinkTopic => (singletonProducer, sinkTopic)),
|
||||
produce = produceOpt.getOrElse(producer),
|
||||
trustStoreLocationOpt = trustStoreLocationOpt,
|
||||
statsReceiver = statsReceiver,
|
||||
decider = decider,
|
||||
zone = zone,
|
||||
maybeProcess = maybeProcess,
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,203 @@
|
||||
package com.twitter.unified_user_actions.service.module
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.decider.SimpleRecipient
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finatra.kafka.producers.BlockingFinagleKafkaProducer
|
||||
import com.twitter.finatra.kafka.serde.ScalaSerdes
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.finatra.kafka.serde.UnKeyedSerde
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.inject.annotations.Flag
|
||||
import org.apache.kafka.clients.consumer.ConsumerRecord
|
||||
import org.apache.kafka.clients.producer.ProducerRecord
|
||||
import org.apache.kafka.common.header.Headers
|
||||
import org.apache.kafka.common.record.CompressionType
|
||||
import com.twitter.kafka.client.headers.Zone
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.unified_user_actions.adapter.AbstractAdapter
|
||||
import com.twitter.unified_user_actions.adapter.uua_aggregates.RekeyUuaAdapter
|
||||
import com.twitter.unified_user_actions.kafka.ClientConfigs
|
||||
import com.twitter.unified_user_actions.kafka.ClientProviders
|
||||
import com.twitter.unified_user_actions.kafka.CompressionTypeFlag
|
||||
import com.twitter.unified_user_actions.kafka.serde.NullableScalaSerdes
|
||||
import com.twitter.unified_user_actions.thriftscala.KeyedUuaTweet
|
||||
import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.Future
|
||||
import com.twitter.util.StorageUnit
|
||||
import com.twitter.util.logging.Logging
|
||||
import javax.inject.Singleton
|
||||
|
||||
object KafkaProcessorRekeyUuaModule extends TwitterModule with Logging {
|
||||
override def modules = Seq(FlagsModule)
|
||||
|
||||
private val adapter = new RekeyUuaAdapter
|
||||
// NOTE: This is a shared processor name in order to simplify monviz stat computation.
|
||||
private final val processorName = "uuaProcessor"
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def providesKafkaProcessor(
|
||||
decider: Decider,
|
||||
@Flag(FlagsModule.cluster) cluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String,
|
||||
@Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String,
|
||||
@Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String],
|
||||
@Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String,
|
||||
@Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String,
|
||||
@Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int,
|
||||
@Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int,
|
||||
@Flag(FlagsModule.commitInterval) commitInterval: Duration,
|
||||
@Flag(FlagsModule.maxPollRecords) maxPollRecords: Int,
|
||||
@Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration,
|
||||
@Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration,
|
||||
@Flag(FlagsModule.fetchMax) fetchMax: StorageUnit,
|
||||
@Flag(FlagsModule.batchSize) batchSize: StorageUnit,
|
||||
@Flag(FlagsModule.linger) linger: Duration,
|
||||
@Flag(FlagsModule.bufferMem) bufferMem: StorageUnit,
|
||||
@Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag,
|
||||
@Flag(FlagsModule.retries) retries: Int,
|
||||
@Flag(FlagsModule.retryBackoff) retryBackoff: Duration,
|
||||
@Flag(FlagsModule.requestTimeout) requestTimeout: Duration,
|
||||
@Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean,
|
||||
@Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String,
|
||||
statsReceiver: StatsReceiver,
|
||||
): AtLeastOnceProcessor[UnKeyed, UnifiedUserAction] = {
|
||||
provideAtLeastOnceProcessor(
|
||||
name = processorName,
|
||||
kafkaSourceCluster = kafkaSourceCluster,
|
||||
kafkaGroupId = kafkaGroupId,
|
||||
kafkaSourceTopic = kafkaSourceTopic,
|
||||
commitInterval = commitInterval,
|
||||
maxPollRecords = maxPollRecords,
|
||||
maxPollInterval = maxPollInterval,
|
||||
sessionTimeout = sessionTimeout,
|
||||
fetchMax = fetchMax,
|
||||
processorMaxPendingRequests = kafkaMaxPendingRequests,
|
||||
processorWorkerThreads = kafkaWorkerThreads,
|
||||
adapter = adapter,
|
||||
kafkaSinkTopics = kafkaSinkTopics,
|
||||
kafkaDestCluster = kafkaDestCluster,
|
||||
kafkaProducerClientId = kafkaProducerClientId,
|
||||
batchSize = batchSize,
|
||||
linger = linger,
|
||||
bufferMem = bufferMem,
|
||||
compressionType = compressionTypeFlag.compressionType,
|
||||
retries = retries,
|
||||
retryBackoff = retryBackoff,
|
||||
requestTimeout = requestTimeout,
|
||||
statsReceiver = statsReceiver,
|
||||
trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None,
|
||||
decider = decider,
|
||||
zone = ZoneFiltering.zoneMapping(cluster),
|
||||
maybeProcess = ZoneFiltering.noFiltering
|
||||
)
|
||||
}
|
||||
|
||||
def producer(
|
||||
producer: BlockingFinagleKafkaProducer[Long, KeyedUuaTweet],
|
||||
k: Long,
|
||||
v: KeyedUuaTweet,
|
||||
sinkTopic: String,
|
||||
headers: Headers,
|
||||
statsReceiver: StatsReceiver,
|
||||
decider: Decider,
|
||||
): Future[Unit] =
|
||||
if (decider.isAvailable(feature = s"RekeyUUA${v.actionType}", Some(SimpleRecipient(k))))
|
||||
// If we were to enable xDC replicator, then we can safely remove the Zone header since xDC
|
||||
// replicator works in the following way:
|
||||
// - If the message does not have a header, the replicator will assume it is local and
|
||||
// set the header, copy the message
|
||||
// - If the message has a header that is the local zone, the replicator will copy the message
|
||||
// - If the message has a header for a different zone, the replicator will drop the message
|
||||
producer
|
||||
.send(new ProducerRecord[Long, KeyedUuaTweet](sinkTopic, null, k, v, headers))
|
||||
.onSuccess { _ => statsReceiver.counter("publishSuccess", sinkTopic).incr() }
|
||||
.onFailure { e: Throwable =>
|
||||
statsReceiver.counter("publishFailure", sinkTopic).incr()
|
||||
error(s"Publish error to topic $sinkTopic: $e")
|
||||
}.unit
|
||||
else Future.Unit
|
||||
|
||||
def provideAtLeastOnceProcessor[K, V](
|
||||
name: String,
|
||||
kafkaSourceCluster: String,
|
||||
kafkaGroupId: String,
|
||||
kafkaSourceTopic: String,
|
||||
commitInterval: Duration = ClientConfigs.kafkaCommitIntervalDefault,
|
||||
maxPollRecords: Int = ClientConfigs.consumerMaxPollRecordsDefault,
|
||||
maxPollInterval: Duration = ClientConfigs.consumerMaxPollIntervalDefault,
|
||||
sessionTimeout: Duration = ClientConfigs.consumerSessionTimeoutDefault,
|
||||
fetchMax: StorageUnit = ClientConfigs.consumerFetchMaxDefault,
|
||||
fetchMin: StorageUnit = ClientConfigs.consumerFetchMinDefault,
|
||||
processorMaxPendingRequests: Int,
|
||||
processorWorkerThreads: Int,
|
||||
adapter: AbstractAdapter[UnifiedUserAction, Long, KeyedUuaTweet],
|
||||
kafkaSinkTopics: Seq[String],
|
||||
kafkaDestCluster: String,
|
||||
kafkaProducerClientId: String,
|
||||
batchSize: StorageUnit = ClientConfigs.producerBatchSizeDefault,
|
||||
linger: Duration = ClientConfigs.producerLingerDefault,
|
||||
bufferMem: StorageUnit = ClientConfigs.producerBufferMemDefault,
|
||||
compressionType: CompressionType = ClientConfigs.compressionDefault.compressionType,
|
||||
retries: Int = ClientConfigs.retriesDefault,
|
||||
retryBackoff: Duration = ClientConfigs.retryBackoffDefault,
|
||||
requestTimeout: Duration = ClientConfigs.producerRequestTimeoutDefault,
|
||||
produceOpt: Option[
|
||||
(BlockingFinagleKafkaProducer[Long, KeyedUuaTweet], Long, KeyedUuaTweet, String, Headers,
|
||||
StatsReceiver, Decider) => Future[Unit]
|
||||
] = Some(producer),
|
||||
trustStoreLocationOpt: Option[String] = Some(ClientConfigs.trustStoreLocationDefault),
|
||||
statsReceiver: StatsReceiver,
|
||||
decider: Decider,
|
||||
zone: Zone,
|
||||
maybeProcess: (ConsumerRecord[UnKeyed, UnifiedUserAction], Zone) => Boolean,
|
||||
): AtLeastOnceProcessor[UnKeyed, UnifiedUserAction] = {
|
||||
|
||||
lazy val singletonProducer = ClientProviders.mkProducer[Long, KeyedUuaTweet](
|
||||
bootstrapServer = kafkaDestCluster,
|
||||
clientId = kafkaProducerClientId,
|
||||
keySerde = ScalaSerdes.Long.serializer,
|
||||
valueSerde = ScalaSerdes.Thrift[KeyedUuaTweet].serializer,
|
||||
idempotence = false,
|
||||
batchSize = batchSize,
|
||||
linger = linger,
|
||||
bufferMem = bufferMem,
|
||||
compressionType = compressionType,
|
||||
retries = retries,
|
||||
retryBackoff = retryBackoff,
|
||||
requestTimeout = requestTimeout,
|
||||
trustStoreLocationOpt = trustStoreLocationOpt,
|
||||
)
|
||||
|
||||
KafkaProcessorProvider.mkAtLeastOnceProcessor[UnKeyed, UnifiedUserAction, Long, KeyedUuaTweet](
|
||||
name = name,
|
||||
kafkaSourceCluster = kafkaSourceCluster,
|
||||
kafkaGroupId = kafkaGroupId,
|
||||
kafkaSourceTopic = kafkaSourceTopic,
|
||||
sourceKeyDeserializer = UnKeyedSerde.deserializer,
|
||||
sourceValueDeserializer = NullableScalaSerdes
|
||||
.Thrift[UnifiedUserAction](statsReceiver.counter("deserializerErrors")).deserializer,
|
||||
commitInterval = commitInterval,
|
||||
maxPollRecords = maxPollRecords,
|
||||
maxPollInterval = maxPollInterval,
|
||||
sessionTimeout = sessionTimeout,
|
||||
fetchMax = fetchMax,
|
||||
fetchMin = fetchMin,
|
||||
processorMaxPendingRequests = processorMaxPendingRequests,
|
||||
processorWorkerThreads = processorWorkerThreads,
|
||||
adapter = adapter,
|
||||
kafkaProducersAndSinkTopics =
|
||||
kafkaSinkTopics.map(sinkTopic => (singletonProducer, sinkTopic)),
|
||||
produce = produceOpt.getOrElse(producer),
|
||||
trustStoreLocationOpt = trustStoreLocationOpt,
|
||||
statsReceiver = statsReceiver,
|
||||
decider = decider,
|
||||
zone = zone,
|
||||
maybeProcess = maybeProcess,
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,88 @@
|
||||
package com.twitter.unified_user_actions.service.module
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.finatra.kafka.serde.UnKeyedSerde
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.inject.annotations.Flag
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.tweetypie.thriftscala.RetweetArchivalEvent
|
||||
import com.twitter.unified_user_actions.adapter.retweet_archival_events.RetweetArchivalEventsAdapter
|
||||
import com.twitter.unified_user_actions.kafka.CompressionTypeFlag
|
||||
import com.twitter.unified_user_actions.kafka.serde.NullableScalaSerdes
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.StorageUnit
|
||||
import com.twitter.util.logging.Logging
|
||||
import javax.inject.Singleton
|
||||
|
||||
object KafkaProcessorRetweetArchivalEventsModule extends TwitterModule with Logging {
|
||||
override def modules = Seq(FlagsModule)
|
||||
|
||||
private val adapter = new RetweetArchivalEventsAdapter
|
||||
// NOTE: This is a shared processor name in order to simplify monviz stat computation.
|
||||
private final val processorName = "uuaProcessor"
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def providesKafkaProcessor(
|
||||
decider: Decider,
|
||||
@Flag(FlagsModule.cluster) cluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String,
|
||||
@Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String,
|
||||
@Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String],
|
||||
@Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String,
|
||||
@Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String,
|
||||
@Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int,
|
||||
@Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int,
|
||||
@Flag(FlagsModule.commitInterval) commitInterval: Duration,
|
||||
@Flag(FlagsModule.maxPollRecords) maxPollRecords: Int,
|
||||
@Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration,
|
||||
@Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration,
|
||||
@Flag(FlagsModule.fetchMax) fetchMax: StorageUnit,
|
||||
@Flag(FlagsModule.batchSize) batchSize: StorageUnit,
|
||||
@Flag(FlagsModule.linger) linger: Duration,
|
||||
@Flag(FlagsModule.bufferMem) bufferMem: StorageUnit,
|
||||
@Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag,
|
||||
@Flag(FlagsModule.retries) retries: Int,
|
||||
@Flag(FlagsModule.retryBackoff) retryBackoff: Duration,
|
||||
@Flag(FlagsModule.requestTimeout) requestTimeout: Duration,
|
||||
@Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean,
|
||||
@Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String,
|
||||
statsReceiver: StatsReceiver,
|
||||
): AtLeastOnceProcessor[UnKeyed, RetweetArchivalEvent] = {
|
||||
KafkaProcessorProvider.provideDefaultAtLeastOnceProcessor(
|
||||
name = processorName,
|
||||
kafkaSourceCluster = kafkaSourceCluster,
|
||||
kafkaGroupId = kafkaGroupId,
|
||||
kafkaSourceTopic = kafkaSourceTopic,
|
||||
sourceKeyDeserializer = UnKeyedSerde.deserializer,
|
||||
sourceValueDeserializer = NullableScalaSerdes
|
||||
.Thrift[RetweetArchivalEvent](statsReceiver.counter("deserializerErrors")).deserializer,
|
||||
commitInterval = commitInterval,
|
||||
maxPollRecords = maxPollRecords,
|
||||
maxPollInterval = maxPollInterval,
|
||||
sessionTimeout = sessionTimeout,
|
||||
fetchMax = fetchMax,
|
||||
processorMaxPendingRequests = kafkaMaxPendingRequests,
|
||||
processorWorkerThreads = kafkaWorkerThreads,
|
||||
adapter = adapter,
|
||||
kafkaSinkTopics = kafkaSinkTopics,
|
||||
kafkaDestCluster = kafkaDestCluster,
|
||||
kafkaProducerClientId = kafkaProducerClientId,
|
||||
batchSize = batchSize,
|
||||
linger = linger,
|
||||
bufferMem = bufferMem,
|
||||
compressionType = compressionTypeFlag.compressionType,
|
||||
retries = retries,
|
||||
retryBackoff = retryBackoff,
|
||||
requestTimeout = requestTimeout,
|
||||
statsReceiver = statsReceiver,
|
||||
trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None,
|
||||
decider = decider,
|
||||
zone = ZoneFiltering.zoneMapping(cluster),
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,90 @@
|
||||
package com.twitter.unified_user_actions.service.module
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.finatra.kafka.serde.UnKeyedSerde
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.inject.annotations.Flag
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.socialgraph.thriftscala.WriteEvent
|
||||
import com.twitter.unified_user_actions.adapter.social_graph_event.SocialGraphAdapter
|
||||
import com.twitter.unified_user_actions.kafka.CompressionTypeFlag
|
||||
import com.twitter.unified_user_actions.kafka.serde.NullableScalaSerdes
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.StorageUnit
|
||||
import com.twitter.util.logging.Logging
|
||||
import javax.inject.Singleton
|
||||
|
||||
class KafkaProcessorSocialGraphModule {}
|
||||
|
||||
object KafkaProcessorSocialGraphModule extends TwitterModule with Logging {
|
||||
override def modules = Seq(FlagsModule)
|
||||
|
||||
private val socialGraphAdapter = new SocialGraphAdapter
|
||||
// NOTE: This is a shared processor name in order to simplify monviz stat computation.
|
||||
private final val processorName = "uuaProcessor"
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def providesKafkaProcessor(
|
||||
decider: Decider,
|
||||
@Flag(FlagsModule.cluster) cluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String,
|
||||
@Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String,
|
||||
@Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String],
|
||||
@Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String,
|
||||
@Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String,
|
||||
@Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int,
|
||||
@Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int,
|
||||
@Flag(FlagsModule.commitInterval) commitInterval: Duration,
|
||||
@Flag(FlagsModule.maxPollRecords) maxPollRecords: Int,
|
||||
@Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration,
|
||||
@Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration,
|
||||
@Flag(FlagsModule.fetchMax) fetchMax: StorageUnit,
|
||||
@Flag(FlagsModule.batchSize) batchSize: StorageUnit,
|
||||
@Flag(FlagsModule.linger) linger: Duration,
|
||||
@Flag(FlagsModule.bufferMem) bufferMem: StorageUnit,
|
||||
@Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag,
|
||||
@Flag(FlagsModule.retries) retries: Int,
|
||||
@Flag(FlagsModule.retryBackoff) retryBackoff: Duration,
|
||||
@Flag(FlagsModule.requestTimeout) requestTimeout: Duration,
|
||||
@Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean,
|
||||
@Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String,
|
||||
statsReceiver: StatsReceiver,
|
||||
): AtLeastOnceProcessor[UnKeyed, WriteEvent] = {
|
||||
KafkaProcessorProvider.provideDefaultAtLeastOnceProcessor(
|
||||
name = processorName,
|
||||
kafkaSourceCluster = kafkaSourceCluster,
|
||||
kafkaGroupId = kafkaGroupId,
|
||||
kafkaSourceTopic = kafkaSourceTopic,
|
||||
sourceKeyDeserializer = UnKeyedSerde.deserializer,
|
||||
sourceValueDeserializer = NullableScalaSerdes
|
||||
.Thrift[WriteEvent](statsReceiver.counter("deserializerErrors")).deserializer,
|
||||
commitInterval = commitInterval,
|
||||
maxPollRecords = maxPollRecords,
|
||||
maxPollInterval = maxPollInterval,
|
||||
sessionTimeout = sessionTimeout,
|
||||
fetchMax = fetchMax,
|
||||
processorMaxPendingRequests = kafkaMaxPendingRequests,
|
||||
processorWorkerThreads = kafkaWorkerThreads,
|
||||
adapter = socialGraphAdapter,
|
||||
kafkaSinkTopics = kafkaSinkTopics,
|
||||
kafkaDestCluster = kafkaDestCluster,
|
||||
kafkaProducerClientId = kafkaProducerClientId,
|
||||
batchSize = batchSize,
|
||||
linger = linger,
|
||||
bufferMem = bufferMem,
|
||||
compressionType = compressionTypeFlag.compressionType,
|
||||
retries = retries,
|
||||
retryBackoff = retryBackoff,
|
||||
requestTimeout = requestTimeout,
|
||||
statsReceiver = statsReceiver,
|
||||
trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None,
|
||||
decider = decider,
|
||||
zone = ZoneFiltering.zoneMapping(cluster),
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,89 @@
|
||||
package com.twitter.unified_user_actions.service.module
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.finatra.kafka.serde.UnKeyedSerde
|
||||
import com.twitter.inject.annotations.Flag
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.timelineservice.thriftscala.ContextualizedFavoriteEvent
|
||||
import com.twitter.unified_user_actions.adapter.tls_favs_event.TlsFavsAdapter
|
||||
import com.twitter.unified_user_actions.kafka.CompressionTypeFlag
|
||||
import com.twitter.unified_user_actions.kafka.serde.NullableScalaSerdes
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.StorageUnit
|
||||
import com.twitter.util.logging.Logging
|
||||
import javax.inject.Singleton
|
||||
|
||||
object KafkaProcessorTlsFavsModule extends TwitterModule with Logging {
|
||||
override def modules = Seq(FlagsModule)
|
||||
|
||||
private val tlsFavsAdapter = new TlsFavsAdapter
|
||||
// NOTE: This is a shared processor name in order to simplify monviz stat computation.
|
||||
private final val processorName = "uuaProcessor"
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def providesKafkaProcessor(
|
||||
decider: Decider,
|
||||
@Flag(FlagsModule.cluster) cluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String,
|
||||
@Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String,
|
||||
@Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String],
|
||||
@Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String,
|
||||
@Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String,
|
||||
@Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int,
|
||||
@Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int,
|
||||
@Flag(FlagsModule.commitInterval) commitInterval: Duration,
|
||||
@Flag(FlagsModule.maxPollRecords) maxPollRecords: Int,
|
||||
@Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration,
|
||||
@Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration,
|
||||
@Flag(FlagsModule.fetchMax) fetchMax: StorageUnit,
|
||||
@Flag(FlagsModule.batchSize) batchSize: StorageUnit,
|
||||
@Flag(FlagsModule.linger) linger: Duration,
|
||||
@Flag(FlagsModule.bufferMem) bufferMem: StorageUnit,
|
||||
@Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag,
|
||||
@Flag(FlagsModule.retries) retries: Int,
|
||||
@Flag(FlagsModule.retryBackoff) retryBackoff: Duration,
|
||||
@Flag(FlagsModule.requestTimeout) requestTimeout: Duration,
|
||||
@Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean,
|
||||
@Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String,
|
||||
statsReceiver: StatsReceiver,
|
||||
): AtLeastOnceProcessor[UnKeyed, ContextualizedFavoriteEvent] = {
|
||||
KafkaProcessorProvider.provideDefaultAtLeastOnceProcessor(
|
||||
name = processorName,
|
||||
kafkaSourceCluster = kafkaSourceCluster,
|
||||
kafkaGroupId = kafkaGroupId,
|
||||
kafkaSourceTopic = kafkaSourceTopic,
|
||||
sourceKeyDeserializer = UnKeyedSerde.deserializer,
|
||||
sourceValueDeserializer = NullableScalaSerdes
|
||||
.Thrift[ContextualizedFavoriteEvent](
|
||||
statsReceiver.counter("deserializerErrors")).deserializer,
|
||||
commitInterval = commitInterval,
|
||||
maxPollRecords = maxPollRecords,
|
||||
maxPollInterval = maxPollInterval,
|
||||
sessionTimeout = sessionTimeout,
|
||||
fetchMax = fetchMax,
|
||||
processorMaxPendingRequests = kafkaMaxPendingRequests,
|
||||
processorWorkerThreads = kafkaWorkerThreads,
|
||||
adapter = tlsFavsAdapter,
|
||||
kafkaSinkTopics = kafkaSinkTopics,
|
||||
kafkaDestCluster = kafkaDestCluster,
|
||||
kafkaProducerClientId = kafkaProducerClientId,
|
||||
batchSize = batchSize,
|
||||
linger = linger,
|
||||
bufferMem = bufferMem,
|
||||
compressionType = compressionTypeFlag.compressionType,
|
||||
retries = retries,
|
||||
retryBackoff = retryBackoff,
|
||||
requestTimeout = requestTimeout,
|
||||
statsReceiver = statsReceiver,
|
||||
trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None,
|
||||
decider = decider,
|
||||
zone = ZoneFiltering.zoneMapping(cluster),
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,90 @@
|
||||
package com.twitter.unified_user_actions.service.module
|
||||
|
||||
import com.google.inject
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.finatra.kafka.serde.UnKeyedSerde
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.inject.annotations.Flag
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.tweetypie.thriftscala.TweetEvent
|
||||
import com.twitter.unified_user_actions.adapter.tweetypie_event.TweetypieEventAdapter
|
||||
import com.twitter.unified_user_actions.kafka.CompressionTypeFlag
|
||||
import com.twitter.unified_user_actions.kafka.serde.NullableScalaSerdes
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.StorageUnit
|
||||
import com.twitter.util.logging.Logging
|
||||
import javax.inject.Singleton
|
||||
|
||||
object KafkaProcessorTweetypieEventModule extends TwitterModule with Logging {
|
||||
override def modules: Seq[inject.Module] = Seq(FlagsModule)
|
||||
|
||||
private val tweetypieEventAdapter = new TweetypieEventAdapter
|
||||
// NOTE: This is a shared processor name in order to simplify monviz stat computation.
|
||||
private final val processorName = "uuaProcessor"
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def providesKafkaProcessor(
|
||||
decider: Decider,
|
||||
@Flag(FlagsModule.cluster) cluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String,
|
||||
@Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String,
|
||||
@Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String],
|
||||
@Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String,
|
||||
@Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String,
|
||||
@Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int,
|
||||
@Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int,
|
||||
@Flag(FlagsModule.commitInterval) commitInterval: Duration,
|
||||
@Flag(FlagsModule.maxPollRecords) maxPollRecords: Int,
|
||||
@Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration,
|
||||
@Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration,
|
||||
@Flag(FlagsModule.fetchMax) fetchMax: StorageUnit,
|
||||
@Flag(FlagsModule.batchSize) batchSize: StorageUnit,
|
||||
@Flag(FlagsModule.linger) linger: Duration,
|
||||
@Flag(FlagsModule.bufferMem) bufferMem: StorageUnit,
|
||||
@Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag,
|
||||
@Flag(FlagsModule.retries) retries: Int,
|
||||
@Flag(FlagsModule.retryBackoff) retryBackoff: Duration,
|
||||
@Flag(FlagsModule.requestTimeout) requestTimeout: Duration,
|
||||
@Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean,
|
||||
@Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String,
|
||||
statsReceiver: StatsReceiver,
|
||||
): AtLeastOnceProcessor[UnKeyed, TweetEvent] = {
|
||||
KafkaProcessorProvider.provideDefaultAtLeastOnceProcessor(
|
||||
name = processorName,
|
||||
kafkaSourceCluster = kafkaSourceCluster,
|
||||
kafkaGroupId = kafkaGroupId,
|
||||
kafkaSourceTopic = kafkaSourceTopic,
|
||||
sourceKeyDeserializer = UnKeyedSerde.deserializer,
|
||||
sourceValueDeserializer = NullableScalaSerdes
|
||||
.Thrift[TweetEvent](statsReceiver.counter("deserializerErrors")).deserializer,
|
||||
commitInterval = commitInterval,
|
||||
maxPollRecords = maxPollRecords,
|
||||
maxPollInterval = maxPollInterval,
|
||||
sessionTimeout = sessionTimeout,
|
||||
fetchMax = fetchMax,
|
||||
processorMaxPendingRequests = kafkaMaxPendingRequests,
|
||||
processorWorkerThreads = kafkaWorkerThreads,
|
||||
adapter = tweetypieEventAdapter,
|
||||
kafkaSinkTopics = kafkaSinkTopics,
|
||||
kafkaDestCluster = kafkaDestCluster,
|
||||
kafkaProducerClientId = kafkaProducerClientId,
|
||||
batchSize = batchSize,
|
||||
linger = linger,
|
||||
bufferMem = bufferMem,
|
||||
compressionType = compressionTypeFlag.compressionType,
|
||||
retries = retries,
|
||||
retryBackoff = retryBackoff,
|
||||
requestTimeout = requestTimeout,
|
||||
statsReceiver = statsReceiver,
|
||||
trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None,
|
||||
decider = decider,
|
||||
zone = ZoneFiltering.zoneMapping(cluster),
|
||||
)
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,87 @@
|
||||
package com.twitter.unified_user_actions.service.module
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.finatra.kafka.serde.UnKeyedSerde
|
||||
import com.twitter.gizmoduck.thriftscala.UserModification
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.inject.annotations.Flag
|
||||
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
|
||||
import com.twitter.unified_user_actions.adapter.user_modification.UserModificationAdapter
|
||||
import com.twitter.unified_user_actions.kafka.CompressionTypeFlag
|
||||
import com.twitter.unified_user_actions.kafka.serde.NullableScalaSerdes
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.StorageUnit
|
||||
import com.twitter.util.logging.Logging
|
||||
import javax.inject.Singleton
|
||||
|
||||
object KafkaProcessorUserModificationModule extends TwitterModule with Logging {
|
||||
override def modules = Seq(FlagsModule)
|
||||
|
||||
// NOTE: This is a shared processor name in order to simplify monviz stat computation.
|
||||
private final val processorName = "uuaProcessor"
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def providesKafkaProcessor(
|
||||
decider: Decider,
|
||||
@Flag(FlagsModule.cluster) cluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String,
|
||||
@Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String,
|
||||
@Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String,
|
||||
@Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String],
|
||||
@Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String,
|
||||
@Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String,
|
||||
@Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int,
|
||||
@Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int,
|
||||
@Flag(FlagsModule.commitInterval) commitInterval: Duration,
|
||||
@Flag(FlagsModule.maxPollRecords) maxPollRecords: Int,
|
||||
@Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration,
|
||||
@Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration,
|
||||
@Flag(FlagsModule.fetchMax) fetchMax: StorageUnit,
|
||||
@Flag(FlagsModule.batchSize) batchSize: StorageUnit,
|
||||
@Flag(FlagsModule.linger) linger: Duration,
|
||||
@Flag(FlagsModule.bufferMem) bufferMem: StorageUnit,
|
||||
@Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag,
|
||||
@Flag(FlagsModule.retries) retries: Int,
|
||||
@Flag(FlagsModule.retryBackoff) retryBackoff: Duration,
|
||||
@Flag(FlagsModule.requestTimeout) requestTimeout: Duration,
|
||||
@Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean,
|
||||
@Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String,
|
||||
statsReceiver: StatsReceiver,
|
||||
): AtLeastOnceProcessor[UnKeyed, UserModification] = {
|
||||
KafkaProcessorProvider.provideDefaultAtLeastOnceProcessor(
|
||||
name = processorName,
|
||||
kafkaSourceCluster = kafkaSourceCluster,
|
||||
kafkaGroupId = kafkaGroupId,
|
||||
kafkaSourceTopic = kafkaSourceTopic,
|
||||
sourceKeyDeserializer = UnKeyedSerde.deserializer,
|
||||
sourceValueDeserializer = NullableScalaSerdes
|
||||
.Thrift[UserModification](statsReceiver.counter("deserializerErrors")).deserializer,
|
||||
commitInterval = commitInterval,
|
||||
maxPollRecords = maxPollRecords,
|
||||
maxPollInterval = maxPollInterval,
|
||||
sessionTimeout = sessionTimeout,
|
||||
fetchMax = fetchMax,
|
||||
processorMaxPendingRequests = kafkaMaxPendingRequests,
|
||||
processorWorkerThreads = kafkaWorkerThreads,
|
||||
adapter = new UserModificationAdapter,
|
||||
kafkaSinkTopics = kafkaSinkTopics,
|
||||
kafkaDestCluster = kafkaDestCluster,
|
||||
kafkaProducerClientId = kafkaProducerClientId,
|
||||
batchSize = batchSize,
|
||||
linger = linger,
|
||||
bufferMem = bufferMem,
|
||||
compressionType = compressionTypeFlag.compressionType,
|
||||
retries = retries,
|
||||
retryBackoff = retryBackoff,
|
||||
requestTimeout = requestTimeout,
|
||||
statsReceiver = statsReceiver,
|
||||
trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None,
|
||||
decider = decider,
|
||||
zone = ZoneFiltering.zoneMapping(cluster),
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,5 @@
|
||||
package com.twitter.unified_user_actions.service.module
|
||||
|
||||
case class TopicsMapping(
|
||||
all: String = "unified_user_actions",
|
||||
engagementsOnly: String = "unified_user_actions_engagements")
|
||||
@ -0,0 +1,22 @@
|
||||
package com.twitter.unified_user_actions.service.module
|
||||
|
||||
import com.twitter.kafka.client.headers.ATLA
|
||||
import com.twitter.kafka.client.headers.Implicits._
|
||||
import com.twitter.kafka.client.headers.PDXA
|
||||
import com.twitter.kafka.client.headers.Zone
|
||||
import org.apache.kafka.clients.consumer.ConsumerRecord
|
||||
|
||||
object ZoneFiltering {
|
||||
def zoneMapping(zone: String): Zone = zone.toLowerCase match {
|
||||
case "atla" => ATLA
|
||||
case "pdxa" => PDXA
|
||||
case _ =>
|
||||
throw new IllegalArgumentException(
|
||||
s"zone must be provided and must be one of [atla,pdxa], provided $zone")
|
||||
}
|
||||
|
||||
def localDCFiltering[K, V](event: ConsumerRecord[K, V], localZone: Zone): Boolean =
|
||||
event.headers().isLocalZone(localZone)
|
||||
|
||||
def noFiltering[K, V](event: ConsumerRecord[K, V], localZone: Zone): Boolean = true
|
||||
}
|
||||
@ -0,0 +1,4 @@
|
||||
resources(
|
||||
sources = ["*.*"],
|
||||
tags = ["bazel-compatible"],
|
||||
)
|
||||
@ -0,0 +1,6 @@
|
||||
PublishServerTweetFav:
|
||||
default_availability: 10000
|
||||
RekeyUUAIesourceClientTweetRenderImpression:
|
||||
default_availability: 10000
|
||||
EnrichmentPlannerSampling:
|
||||
default_availability: 10000
|
||||
45
unified_user_actions/service/src/test/resources/logback.xml
Normal file
45
unified_user_actions/service/src/test/resources/logback.xml
Normal file
@ -0,0 +1,45 @@
|
||||
<configuration>
|
||||
<!-- ===================================================== -->
|
||||
<!-- Console appender for local debugging and testing -->
|
||||
<!-- ===================================================== -->
|
||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<encoder>
|
||||
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<!-- ===================================================== -->
|
||||
<!-- Package Config -->
|
||||
<!-- ===================================================== -->
|
||||
|
||||
<!-- Root Config -->
|
||||
<root level="info">
|
||||
<appender-ref ref="STDOUT"/>
|
||||
</root>
|
||||
|
||||
<!-- Per-Package Config -->
|
||||
<logger name="com.twitter" level="info"/>
|
||||
<logger name="com.twitter.zookeeper.client.internal" level="warn"/>
|
||||
<logger name="com.twitter.zookeeper.client.internal.ClientCnxnSocket" level="error"/>
|
||||
<logger name="com.twitter.logging.ScribeHandler" level="warn"/>
|
||||
<logger name="com.twitter.finatra" level="info"/>
|
||||
|
||||
<logger name="org.apache.kafka" level="debug"/>
|
||||
<logger name="org.apache.kafka.clients" level="info"/>
|
||||
<logger name="org.apache.kafka.clients.NetworkClient" level="warn"/>
|
||||
<logger name="org.apache.kafka.clients.consumer.internals" level="info"/>
|
||||
<logger name="org.apache.kafka.common.network" level="warn" />
|
||||
<logger name="org.apache.kafka.common.security.authenticator" level="info" />
|
||||
<logger name="kafka.server.KafkaConfig" level="off" />
|
||||
<logger name="org.apache.kafka.clients.producer.ProducerConfig" level="off" />
|
||||
<logger name="org.apache.kafka.clients.consumer.ConsumerConfig" level="off" />
|
||||
<logger name="org.apache.kafka.clients.admin.AdminClientConfig" level="off" />
|
||||
<logger name="org.apache.kafka.common.utils.AppInfoParser" level="off" />
|
||||
<logger name="org.apache.zookeeper" level="off" />
|
||||
|
||||
|
||||
<logger name="com.google.inject" level="info"/>
|
||||
<logger name="io.netty" level="info"/>
|
||||
<logger name="jdk.event" level="info"/>
|
||||
<logger name="javax.security" level="info"/>
|
||||
</configuration>
|
||||
@ -0,0 +1,21 @@
|
||||
junit_tests(
|
||||
name = "tests",
|
||||
sources = ["*.scala"],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/ch/qos/logback:logback-classic",
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"decider/src/main/scala",
|
||||
"kafka/finagle-kafka/finatra-kafka-streams/kafka-streams/src/test/scala:test-deps",
|
||||
"kafka/finagle-kafka/finatra-kafka/src/test/scala:test-deps",
|
||||
"unified_user_actions/enricher/src/main/thrift/com/twitter/unified_user_actions/enricher/internal:internal-scala",
|
||||
"unified_user_actions/enricher/src/test/scala/com/twitter/unified_user_actions/enricher:fixture",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:client-event",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:enrichment-planner",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:rekey-uua-iesource",
|
||||
"unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service:tls-favs",
|
||||
"unified_user_actions/service/src/test/resources",
|
||||
"util/util-mock/src/main/scala/com/twitter/util/mock",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,141 @@
|
||||
package com.twitter.unified_user_actions.service
|
||||
|
||||
import com.google.inject.Stage
|
||||
import com.twitter.app.GlobalFlag
|
||||
import com.twitter.clientapp.thriftscala.EventDetails
|
||||
import com.twitter.clientapp.thriftscala.EventNamespace
|
||||
import com.twitter.clientapp.thriftscala.Item
|
||||
import com.twitter.clientapp.thriftscala.ItemType
|
||||
import com.twitter.clientapp.thriftscala.LogEvent
|
||||
import com.twitter.finatra.kafka.consumers.FinagleKafkaConsumerBuilder
|
||||
import com.twitter.finatra.kafka.domain.AckMode
|
||||
import com.twitter.finatra.kafka.domain.KafkaGroupId
|
||||
import com.twitter.finatra.kafka.domain.KafkaTopic
|
||||
import com.twitter.finatra.kafka.domain.SeekStrategy
|
||||
import com.twitter.finatra.kafka.producers.FinagleKafkaProducerBuilder
|
||||
import com.twitter.finatra.kafka.serde.ScalaSerdes
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.finatra.kafka.serde.UnKeyedSerde
|
||||
import com.twitter.finatra.kafka.test.KafkaFeatureTest
|
||||
import com.twitter.inject.server.EmbeddedTwitterServer
|
||||
import com.twitter.kafka.client.processor.KafkaConsumerClient
|
||||
import com.twitter.logbase.thriftscala.LogBase
|
||||
import com.twitter.unified_user_actions.kafka.ClientConfigs
|
||||
import com.twitter.unified_user_actions.service.module.KafkaProcessorClientEventModule
|
||||
import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.StorageUnit
|
||||
|
||||
class ClientEventServiceStartupTest extends KafkaFeatureTest {
|
||||
private val inputTopic =
|
||||
kafkaTopic(UnKeyedSerde, ScalaSerdes.Thrift[LogEvent], name = "source")
|
||||
private val outputTopic =
|
||||
kafkaTopic(UnKeyedSerde, ScalaSerdes.Thrift[UnifiedUserAction], name = "sink")
|
||||
|
||||
val startupFlags = Map(
|
||||
"kafka.group.id" -> "client-event",
|
||||
"kafka.producer.client.id" -> "uua",
|
||||
"kafka.source.topic" -> inputTopic.topic,
|
||||
"kafka.sink.topics" -> outputTopic.topic,
|
||||
"kafka.consumer.fetch.min" -> "6.megabytes",
|
||||
"kafka.max.pending.requests" -> "100",
|
||||
"kafka.worker.threads" -> "1",
|
||||
"kafka.trust.store.enable" -> "false",
|
||||
"kafka.producer.batch.size" -> "0.byte",
|
||||
"cluster" -> "atla",
|
||||
)
|
||||
|
||||
val deciderFlags = Map(
|
||||
"decider.base" -> "/decider.yml"
|
||||
)
|
||||
|
||||
override protected def kafkaBootstrapFlag: Map[String, String] = {
|
||||
Map(
|
||||
ClientConfigs.kafkaBootstrapServerConfig -> kafkaCluster.bootstrapServers(),
|
||||
ClientConfigs.kafkaBootstrapServerRemoteDestConfig -> kafkaCluster.bootstrapServers(),
|
||||
)
|
||||
}
|
||||
|
||||
override val server: EmbeddedTwitterServer = new EmbeddedTwitterServer(
|
||||
twitterServer = new ClientEventService() {
|
||||
override def warmup(): Unit = {
|
||||
// noop
|
||||
}
|
||||
|
||||
override val overrideModules = Seq(
|
||||
KafkaProcessorClientEventModule
|
||||
)
|
||||
},
|
||||
globalFlags = Map[GlobalFlag[_], String](
|
||||
com.twitter.finatra.kafka.consumers.enableTlsAndKerberos -> "false",
|
||||
),
|
||||
flags = startupFlags ++ kafkaBootstrapFlag ++ deciderFlags,
|
||||
stage = Stage.PRODUCTION
|
||||
)
|
||||
|
||||
private def getConsumer(
|
||||
seekStrategy: SeekStrategy = SeekStrategy.BEGINNING,
|
||||
) = {
|
||||
val builder = FinagleKafkaConsumerBuilder()
|
||||
.dest(brokers.map(_.brokerList()).mkString(","))
|
||||
.clientId("consumer")
|
||||
.groupId(KafkaGroupId("validator"))
|
||||
.keyDeserializer(UnKeyedSerde.deserializer)
|
||||
.valueDeserializer(ScalaSerdes.Thrift[LogEvent].deserializer)
|
||||
.requestTimeout(Duration.fromSeconds(1))
|
||||
.enableAutoCommit(false)
|
||||
.seekStrategy(seekStrategy)
|
||||
|
||||
new KafkaConsumerClient(builder.config)
|
||||
}
|
||||
|
||||
private def getProducer(clientId: String = "producer") = {
|
||||
FinagleKafkaProducerBuilder()
|
||||
.dest(brokers.map(_.brokerList()).mkString(","))
|
||||
.clientId(clientId)
|
||||
.ackMode(AckMode.ALL)
|
||||
.batchSize(StorageUnit.zero)
|
||||
.keySerializer(UnKeyedSerde.serializer)
|
||||
.valueSerializer(ScalaSerdes.Thrift[LogEvent].serializer)
|
||||
.build()
|
||||
}
|
||||
|
||||
test("ClientEventService starts") {
|
||||
server.assertHealthy()
|
||||
}
|
||||
|
||||
test("ClientEventService should process input events") {
|
||||
val producer = getProducer()
|
||||
val inputConsumer = getConsumer()
|
||||
|
||||
val value: LogEvent = LogEvent(
|
||||
eventName = "test_tweet_render_impression_event",
|
||||
eventNamespace =
|
||||
Some(EventNamespace(component = Some("stream"), element = None, action = Some("results"))),
|
||||
eventDetails = Some(
|
||||
EventDetails(
|
||||
items = Some(
|
||||
Seq[Item](
|
||||
Item(id = Some(1L), itemType = Some(ItemType.Tweet))
|
||||
))
|
||||
)),
|
||||
logBase = Some(LogBase(timestamp = 10001L, transactionId = "", ipAddress = ""))
|
||||
)
|
||||
|
||||
try {
|
||||
server.assertHealthy()
|
||||
|
||||
// before, should be empty
|
||||
inputConsumer.subscribe(Set(KafkaTopic(inputTopic.topic)))
|
||||
assert(inputConsumer.poll().count() == 0)
|
||||
|
||||
// after, should contain at least a message
|
||||
await(producer.send(inputTopic.topic, new UnKeyed, value, System.currentTimeMillis))
|
||||
producer.flush()
|
||||
assert(inputConsumer.poll().count() >= 1)
|
||||
} finally {
|
||||
await(producer.close())
|
||||
inputConsumer.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,75 @@
|
||||
package com.twitter.unified_user_actions.service
|
||||
|
||||
import com.twitter.decider.MockDecider
|
||||
import com.twitter.inject.Test
|
||||
import com.twitter.unified_user_actions.service.module.ClientEventDeciderUtils
|
||||
import com.twitter.unified_user_actions.service.module.DefaultDeciderUtils
|
||||
import com.twitter.unified_user_actions.thriftscala._
|
||||
import com.twitter.util.Time
|
||||
import com.twitter.util.mock.Mockito
|
||||
import org.junit.runner.RunWith
|
||||
import org.scalatestplus.junit.JUnitRunner
|
||||
|
||||
@RunWith(classOf[JUnitRunner])
|
||||
class DeciderUtilsTest extends Test with Mockito {
|
||||
trait Fixture {
|
||||
val frozenTime = Time.fromMilliseconds(1658949273000L)
|
||||
|
||||
val publishActionTypes =
|
||||
Set[ActionType](ActionType.ServerTweetFav, ActionType.ClientTweetRenderImpression)
|
||||
|
||||
def decider(
|
||||
features: Set[String] = publishActionTypes.map { action =>
|
||||
s"Publish${action.name}"
|
||||
}
|
||||
) = new MockDecider(features = features)
|
||||
|
||||
def mkUUA(actionType: ActionType) = UnifiedUserAction(
|
||||
userIdentifier = UserIdentifier(userId = Some(91L)),
|
||||
item = Item.TweetInfo(
|
||||
TweetInfo(
|
||||
actionTweetId = 1L,
|
||||
actionTweetAuthorInfo = Some(AuthorInfo(authorId = Some(101L))),
|
||||
)
|
||||
),
|
||||
actionType = actionType,
|
||||
eventMetadata = EventMetadata(
|
||||
sourceTimestampMs = 1001L,
|
||||
receivedTimestampMs = frozenTime.inMilliseconds,
|
||||
sourceLineage = SourceLineage.ServerTlsFavs,
|
||||
traceId = Some(31L)
|
||||
)
|
||||
)
|
||||
|
||||
val uuaServerTweetFav = mkUUA(ActionType.ServerTweetFav)
|
||||
val uuaClientTweetFav = mkUUA(ActionType.ClientTweetFav)
|
||||
val uuaClientTweetRenderImpression = mkUUA(ActionType.ClientTweetRenderImpression)
|
||||
}
|
||||
|
||||
test("Decider Utils") {
|
||||
new Fixture {
|
||||
Time.withTimeAt(frozenTime) { _ =>
|
||||
DefaultDeciderUtils.shouldPublish(
|
||||
decider = decider(),
|
||||
uua = uuaServerTweetFav,
|
||||
sinkTopic = "") shouldBe true
|
||||
DefaultDeciderUtils.shouldPublish(
|
||||
decider = decider(),
|
||||
uua = uuaClientTweetFav,
|
||||
sinkTopic = "") shouldBe false
|
||||
ClientEventDeciderUtils.shouldPublish(
|
||||
decider = decider(),
|
||||
uua = uuaClientTweetRenderImpression,
|
||||
sinkTopic = "unified_user_actions_engagements") shouldBe false
|
||||
ClientEventDeciderUtils.shouldPublish(
|
||||
decider = decider(),
|
||||
uua = uuaClientTweetFav,
|
||||
sinkTopic = "unified_user_actions_engagements") shouldBe false
|
||||
ClientEventDeciderUtils.shouldPublish(
|
||||
decider = decider(features = Set[String](s"Publish${ActionType.ClientTweetFav.name}")),
|
||||
uua = uuaClientTweetFav,
|
||||
sinkTopic = "unified_user_actions_engagements") shouldBe true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,141 @@
|
||||
package com.twitter.unified_user_actions.service
|
||||
|
||||
import com.twitter.finatra.kafka.serde.ScalaSerdes
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.finatra.kafka.serde.UnKeyedSerde
|
||||
import com.twitter.finatra.kafka.test.EmbeddedKafka
|
||||
import com.twitter.finatra.kafkastreams.test.FinatraTopologyTester
|
||||
import com.twitter.finatra.kafkastreams.test.TopologyFeatureTest
|
||||
import com.twitter.unified_user_actions.enricher.EnricherFixture
|
||||
import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentEnvelop
|
||||
import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentIdType
|
||||
import com.twitter.unified_user_actions.enricher.internal.thriftscala.EnrichmentKey
|
||||
import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction
|
||||
import org.apache.kafka.clients.consumer.ConsumerRecord
|
||||
import org.joda.time.DateTime
|
||||
|
||||
/**
|
||||
* This is to test the logic where the service reads and outputs to the same Kafka cluster
|
||||
*/
|
||||
class EnrichmentPlannerServiceTest extends TopologyFeatureTest {
|
||||
val startTime = new DateTime("2022-10-01T00:00:00Z")
|
||||
|
||||
override protected lazy val topologyTester: FinatraTopologyTester = FinatraTopologyTester(
|
||||
"enrichment-planner-tester",
|
||||
new EnrichmentPlannerService,
|
||||
startingWallClockTime = startTime,
|
||||
flags = Map(
|
||||
"decider.base" -> "/decider.yml",
|
||||
"kafka.output.server" -> ""
|
||||
)
|
||||
)
|
||||
|
||||
private val inputTopic = topologyTester.topic(
|
||||
name = EnrichmentPlannerServiceMain.InputTopic,
|
||||
keySerde = UnKeyedSerde,
|
||||
valSerde = ScalaSerdes.Thrift[UnifiedUserAction]
|
||||
)
|
||||
|
||||
private val outputTopic = topologyTester.topic(
|
||||
name = EnrichmentPlannerServiceMain.OutputPartitionedTopic,
|
||||
keySerde = ScalaSerdes.Thrift[EnrichmentKey],
|
||||
valSerde = ScalaSerdes.Thrift[EnrichmentEnvelop]
|
||||
)
|
||||
|
||||
test("can filter unsupported events") {
|
||||
new EnricherFixture {
|
||||
(1L to 10L).foreach(id => {
|
||||
inputTopic.pipeInput(UnKeyed, mkUUAProfileEvent(id))
|
||||
})
|
||||
|
||||
assert(outputTopic.readAllOutput().size === 0)
|
||||
}
|
||||
}
|
||||
|
||||
test("partition key serialization should be correct") {
|
||||
val key = EnrichmentKey(EnrichmentIdType.TweetId, 9999L)
|
||||
val serializer = ScalaSerdes.Thrift[EnrichmentKey].serializer
|
||||
|
||||
val actual = serializer.serialize("test", key)
|
||||
val expected = Array[Byte](8, 0, 1, 0, 0, 0, 0, 10, 0, 2, 0, 0, 0, 0, 0, 0, 39, 15, 0)
|
||||
|
||||
assert(actual.deep === expected.deep)
|
||||
}
|
||||
|
||||
test("partitioned enrichment tweet event is constructed correctly") {
|
||||
new EnricherFixture {
|
||||
val expected = mkUUATweetEvent(888L)
|
||||
inputTopic.pipeInput(UnKeyed, expected)
|
||||
|
||||
val actual = outputTopic.readAllOutput().head
|
||||
|
||||
assert(actual.key() === EnrichmentKey(EnrichmentIdType.TweetId, 888L))
|
||||
assert(
|
||||
actual
|
||||
.value() === EnrichmentEnvelop(
|
||||
expected.hashCode,
|
||||
expected,
|
||||
plan = tweetInfoEnrichmentPlan
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
test("partitioned enrichment tweet notification event is constructed correctly") {
|
||||
new EnricherFixture {
|
||||
val expected = mkUUATweetNotificationEvent(8989L)
|
||||
inputTopic.pipeInput(UnKeyed, expected)
|
||||
|
||||
val actual = outputTopic.readAllOutput().head
|
||||
|
||||
assert(actual.key() === EnrichmentKey(EnrichmentIdType.TweetId, 8989L))
|
||||
assert(
|
||||
actual
|
||||
.value() === EnrichmentEnvelop(
|
||||
expected.hashCode,
|
||||
expected,
|
||||
plan = tweetNotificationEnrichmentPlan
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This is tests the bootstrap server logic in prod. Don't add any new tests here since it is slow.
|
||||
* Use the tests above which is much quicker to be executed and and test the majority of prod logic.
|
||||
*/
|
||||
class EnrichmentPlannerServiceEmbeddedKafkaTest extends TopologyFeatureTest with EmbeddedKafka {
|
||||
val startTime = new DateTime("2022-10-01T00:00:00Z")
|
||||
|
||||
override protected lazy val topologyTester: FinatraTopologyTester = FinatraTopologyTester(
|
||||
"enrichment-planner-tester",
|
||||
new EnrichmentPlannerService,
|
||||
startingWallClockTime = startTime,
|
||||
flags = Map(
|
||||
"decider.base" -> "/decider.yml",
|
||||
"kafka.output.server" -> kafkaCluster.bootstrapServers(),
|
||||
"kafka.output.enable.tls" -> "false"
|
||||
)
|
||||
)
|
||||
|
||||
private lazy val inputTopic = topologyTester.topic(
|
||||
name = EnrichmentPlannerServiceMain.InputTopic,
|
||||
keySerde = UnKeyedSerde,
|
||||
valSerde = ScalaSerdes.Thrift[UnifiedUserAction]
|
||||
)
|
||||
|
||||
private val outputTopic = kafkaTopic(
|
||||
name = EnrichmentPlannerServiceMain.OutputPartitionedTopic,
|
||||
keySerde = ScalaSerdes.Thrift[EnrichmentKey],
|
||||
valSerde = ScalaSerdes.Thrift[EnrichmentEnvelop]
|
||||
)
|
||||
|
||||
test("toCluster should output to expected topic & embeded cluster") {
|
||||
new EnricherFixture {
|
||||
inputTopic.pipeInput(UnKeyed, mkUUATweetEvent(tweetId = 1))
|
||||
val records: Seq[ConsumerRecord[Array[Byte], Array[Byte]]] = outputTopic.consumeRecords(1)
|
||||
|
||||
assert(records.size === 1)
|
||||
assert(records.head.topic() == EnrichmentPlannerServiceMain.OutputPartitionedTopic)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,173 @@
|
||||
package com.twitter.unified_user_actions.service
|
||||
|
||||
import com.google.inject.Stage
|
||||
import com.twitter.adserver.thriftscala.DisplayLocation
|
||||
import com.twitter.app.GlobalFlag
|
||||
import com.twitter.finatra.kafka.consumers.FinagleKafkaConsumerBuilder
|
||||
import com.twitter.finatra.kafka.domain.AckMode
|
||||
import com.twitter.finatra.kafka.domain.KafkaGroupId
|
||||
import com.twitter.finatra.kafka.domain.KafkaTopic
|
||||
import com.twitter.finatra.kafka.domain.SeekStrategy
|
||||
import com.twitter.finatra.kafka.producers.FinagleKafkaProducerBuilder
|
||||
import com.twitter.finatra.kafka.serde.ScalaSerdes
|
||||
import com.twitter.finatra.kafka.serde.UnKeyedSerde
|
||||
import com.twitter.finatra.kafka.test.KafkaFeatureTest
|
||||
import com.twitter.iesource.thriftscala.ClientEventContext
|
||||
import com.twitter.iesource.thriftscala.TweetImpression
|
||||
import com.twitter.iesource.thriftscala.ClientType
|
||||
import com.twitter.iesource.thriftscala.ContextualEventNamespace
|
||||
import com.twitter.iesource.thriftscala.EngagingContext
|
||||
import com.twitter.iesource.thriftscala.EventSource
|
||||
import com.twitter.iesource.thriftscala.InteractionDetails
|
||||
import com.twitter.iesource.thriftscala.InteractionEvent
|
||||
import com.twitter.iesource.thriftscala.InteractionType
|
||||
import com.twitter.iesource.thriftscala.InteractionTargetType
|
||||
import com.twitter.iesource.thriftscala.UserIdentifier
|
||||
import com.twitter.inject.server.EmbeddedTwitterServer
|
||||
import com.twitter.kafka.client.processor.KafkaConsumerClient
|
||||
import com.twitter.unified_user_actions.kafka.ClientConfigs
|
||||
import com.twitter.unified_user_actions.service.module.KafkaProcessorRekeyUuaIesourceModule
|
||||
import com.twitter.unified_user_actions.thriftscala.KeyedUuaTweet
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.StorageUnit
|
||||
|
||||
class RekeyUuaIesourceServiceStartupTest extends KafkaFeatureTest {
|
||||
private val inputTopic =
|
||||
kafkaTopic(ScalaSerdes.Long, ScalaSerdes.CompactThrift[InteractionEvent], name = "source")
|
||||
private val outputTopic =
|
||||
kafkaTopic(ScalaSerdes.Long, ScalaSerdes.Thrift[KeyedUuaTweet], name = "sink")
|
||||
|
||||
val startupFlags = Map(
|
||||
"kafka.group.id" -> "client-event",
|
||||
"kafka.producer.client.id" -> "uua",
|
||||
"kafka.source.topic" -> inputTopic.topic,
|
||||
"kafka.sink.topics" -> outputTopic.topic,
|
||||
"kafka.consumer.fetch.min" -> "6.megabytes",
|
||||
"kafka.max.pending.requests" -> "100",
|
||||
"kafka.worker.threads" -> "1",
|
||||
"kafka.trust.store.enable" -> "false",
|
||||
"kafka.producer.batch.size" -> "0.byte",
|
||||
"cluster" -> "atla",
|
||||
)
|
||||
|
||||
val deciderFlags = Map(
|
||||
"decider.base" -> "/decider.yml"
|
||||
)
|
||||
|
||||
override protected def kafkaBootstrapFlag: Map[String, String] = {
|
||||
Map(
|
||||
ClientConfigs.kafkaBootstrapServerConfig -> kafkaCluster.bootstrapServers(),
|
||||
ClientConfigs.kafkaBootstrapServerRemoteDestConfig -> kafkaCluster.bootstrapServers(),
|
||||
)
|
||||
}
|
||||
|
||||
override val server: EmbeddedTwitterServer = new EmbeddedTwitterServer(
|
||||
twitterServer = new RekeyUuaIesourceService() {
|
||||
override def warmup(): Unit = {
|
||||
// noop
|
||||
}
|
||||
|
||||
override val overrideModules = Seq(
|
||||
KafkaProcessorRekeyUuaIesourceModule
|
||||
)
|
||||
},
|
||||
globalFlags = Map[GlobalFlag[_], String](
|
||||
com.twitter.finatra.kafka.consumers.enableTlsAndKerberos -> "false",
|
||||
),
|
||||
flags = startupFlags ++ kafkaBootstrapFlag ++ deciderFlags,
|
||||
stage = Stage.PRODUCTION
|
||||
)
|
||||
|
||||
private def getConsumer(
|
||||
seekStrategy: SeekStrategy = SeekStrategy.BEGINNING,
|
||||
) = {
|
||||
val builder = FinagleKafkaConsumerBuilder()
|
||||
.dest(brokers.map(_.brokerList()).mkString(","))
|
||||
.clientId("consumer")
|
||||
.groupId(KafkaGroupId("validator"))
|
||||
.keyDeserializer(ScalaSerdes.Long.deserializer)
|
||||
.valueDeserializer(ScalaSerdes.CompactThrift[InteractionEvent].deserializer)
|
||||
.requestTimeout(Duration.fromSeconds(1))
|
||||
.enableAutoCommit(false)
|
||||
.seekStrategy(seekStrategy)
|
||||
|
||||
new KafkaConsumerClient(builder.config)
|
||||
}
|
||||
|
||||
private def getUUAConsumer(
|
||||
seekStrategy: SeekStrategy = SeekStrategy.BEGINNING,
|
||||
) = {
|
||||
val builder = FinagleKafkaConsumerBuilder()
|
||||
.dest(brokers.map(_.brokerList()).mkString(","))
|
||||
.clientId("consumer_uua")
|
||||
.groupId(KafkaGroupId("validator_uua"))
|
||||
.keyDeserializer(UnKeyedSerde.deserializer)
|
||||
.valueDeserializer(ScalaSerdes.Thrift[KeyedUuaTweet].deserializer)
|
||||
.requestTimeout(Duration.fromSeconds(1))
|
||||
.enableAutoCommit(false)
|
||||
.seekStrategy(seekStrategy)
|
||||
|
||||
new KafkaConsumerClient(builder.config)
|
||||
}
|
||||
|
||||
private def getProducer(clientId: String = "producer") = {
|
||||
FinagleKafkaProducerBuilder()
|
||||
.dest(brokers.map(_.brokerList()).mkString(","))
|
||||
.clientId(clientId)
|
||||
.ackMode(AckMode.ALL)
|
||||
.batchSize(StorageUnit.zero)
|
||||
.keySerializer(ScalaSerdes.Long.serializer)
|
||||
.valueSerializer(ScalaSerdes.CompactThrift[InteractionEvent].serializer)
|
||||
.build()
|
||||
}
|
||||
|
||||
test("RekeyUuaIesourceService starts") {
|
||||
server.assertHealthy()
|
||||
}
|
||||
|
||||
test("RekeyUuaIesourceService should process input events") {
|
||||
val producer = getProducer()
|
||||
val inputConsumer = getConsumer()
|
||||
val uuaConsumer = getUUAConsumer()
|
||||
|
||||
val value: InteractionEvent = InteractionEvent(
|
||||
targetId = 1L,
|
||||
targetType = InteractionTargetType.Tweet,
|
||||
engagingUserId = 11L,
|
||||
eventSource = EventSource.ClientEvent,
|
||||
timestampMillis = 123456L,
|
||||
interactionType = Some(InteractionType.TweetRenderImpression),
|
||||
details = InteractionDetails.TweetRenderImpression(TweetImpression()),
|
||||
additionalEngagingUserIdentifiers = UserIdentifier(),
|
||||
engagingContext = EngagingContext.ClientEventContext(
|
||||
ClientEventContext(
|
||||
clientEventNamespace = ContextualEventNamespace(),
|
||||
clientType = ClientType.Iphone,
|
||||
displayLocation = DisplayLocation(1)))
|
||||
)
|
||||
|
||||
try {
|
||||
server.assertHealthy()
|
||||
|
||||
// before, should be empty
|
||||
inputConsumer.subscribe(Set(KafkaTopic(inputTopic.topic)))
|
||||
assert(inputConsumer.poll().count() == 0)
|
||||
|
||||
// after, should contain at least a message
|
||||
await(producer.send(inputTopic.topic, value.targetId, value, System.currentTimeMillis))
|
||||
producer.flush()
|
||||
assert(inputConsumer.poll().count() == 1)
|
||||
|
||||
uuaConsumer.subscribe(Set(KafkaTopic(outputTopic.topic)))
|
||||
// This is tricky: it is not guaranteed that the srvice can process and output the
|
||||
// event to output topic faster than the below consumer. So we'd use a timer here which may
|
||||
// not be the best practice.
|
||||
// If someone finds the below test is flaky, please just remove the below test completely.
|
||||
Thread.sleep(5000L)
|
||||
assert(uuaConsumer.poll().count() == 1)
|
||||
} finally {
|
||||
await(producer.close())
|
||||
inputConsumer.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,153 @@
|
||||
package com.twitter.unified_user_actions.service
|
||||
|
||||
import com.google.inject.Stage
|
||||
import com.twitter.app.GlobalFlag
|
||||
import com.twitter.finatra.kafka.consumers.FinagleKafkaConsumerBuilder
|
||||
import com.twitter.finatra.kafka.domain.AckMode
|
||||
import com.twitter.finatra.kafka.domain.KafkaGroupId
|
||||
import com.twitter.finatra.kafka.domain.KafkaTopic
|
||||
import com.twitter.finatra.kafka.domain.SeekStrategy
|
||||
import com.twitter.finatra.kafka.producers.FinagleKafkaProducerBuilder
|
||||
import com.twitter.finatra.kafka.serde.ScalaSerdes
|
||||
import com.twitter.finatra.kafka.serde.UnKeyed
|
||||
import com.twitter.finatra.kafka.serde.UnKeyedSerde
|
||||
import com.twitter.finatra.kafka.test.KafkaFeatureTest
|
||||
import com.twitter.inject.server.EmbeddedTwitterServer
|
||||
import com.twitter.kafka.client.processor.KafkaConsumerClient
|
||||
import com.twitter.timelineservice.thriftscala.ContextualizedFavoriteEvent
|
||||
import com.twitter.timelineservice.thriftscala.FavoriteEvent
|
||||
import com.twitter.timelineservice.thriftscala.FavoriteEventUnion
|
||||
import com.twitter.timelineservice.thriftscala.LogEventContext
|
||||
import com.twitter.unified_user_actions.kafka.ClientConfigs
|
||||
import com.twitter.unified_user_actions.service.module.KafkaProcessorTlsFavsModule
|
||||
import com.twitter.unified_user_actions.thriftscala.UnifiedUserAction
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.StorageUnit
|
||||
|
||||
class TlsFavServiceStartupTest extends KafkaFeatureTest {
|
||||
private val inputTopic =
|
||||
kafkaTopic(UnKeyedSerde, ScalaSerdes.Thrift[ContextualizedFavoriteEvent], name = "source")
|
||||
private val outputTopic =
|
||||
kafkaTopic(UnKeyedSerde, ScalaSerdes.Thrift[UnifiedUserAction], name = "sink")
|
||||
|
||||
val startupFlags = Map(
|
||||
"kafka.group.id" -> "tls",
|
||||
"kafka.producer.client.id" -> "uua",
|
||||
"kafka.source.topic" -> inputTopic.topic,
|
||||
"kafka.sink.topics" -> outputTopic.topic,
|
||||
"kafka.max.pending.requests" -> "100",
|
||||
"kafka.worker.threads" -> "1",
|
||||
"kafka.trust.store.enable" -> "false",
|
||||
"kafka.producer.batch.size" -> "0.byte",
|
||||
"cluster" -> "atla",
|
||||
)
|
||||
|
||||
val deciderFlags = Map(
|
||||
"decider.base" -> "/decider.yml"
|
||||
)
|
||||
|
||||
override protected def kafkaBootstrapFlag: Map[String, String] = {
|
||||
Map(
|
||||
ClientConfigs.kafkaBootstrapServerConfig -> kafkaCluster.bootstrapServers(),
|
||||
ClientConfigs.kafkaBootstrapServerRemoteDestConfig -> kafkaCluster.bootstrapServers(),
|
||||
)
|
||||
}
|
||||
|
||||
override val server: EmbeddedTwitterServer = new EmbeddedTwitterServer(
|
||||
twitterServer = new TlsFavsService() {
|
||||
override def warmup(): Unit = {
|
||||
// noop
|
||||
}
|
||||
|
||||
override val overrideModules = Seq(
|
||||
KafkaProcessorTlsFavsModule
|
||||
)
|
||||
},
|
||||
globalFlags = Map[GlobalFlag[_], String](
|
||||
com.twitter.finatra.kafka.consumers.enableTlsAndKerberos -> "false",
|
||||
),
|
||||
flags = startupFlags ++ kafkaBootstrapFlag ++ deciderFlags,
|
||||
stage = Stage.PRODUCTION
|
||||
)
|
||||
|
||||
private def getConsumer(
|
||||
seekStrategy: SeekStrategy = SeekStrategy.BEGINNING,
|
||||
) = {
|
||||
val builder = FinagleKafkaConsumerBuilder()
|
||||
.dest(brokers.map(_.brokerList()).mkString(","))
|
||||
.clientId("consumer")
|
||||
.groupId(KafkaGroupId("validator"))
|
||||
.keyDeserializer(UnKeyedSerde.deserializer)
|
||||
.valueDeserializer(ScalaSerdes.Thrift[ContextualizedFavoriteEvent].deserializer)
|
||||
.requestTimeout(Duration.fromSeconds(1))
|
||||
.enableAutoCommit(false)
|
||||
.seekStrategy(seekStrategy)
|
||||
|
||||
new KafkaConsumerClient(builder.config)
|
||||
}
|
||||
|
||||
private def getProducer(clientId: String = "producer") = {
|
||||
FinagleKafkaProducerBuilder()
|
||||
.dest(brokers.map(_.brokerList()).mkString(","))
|
||||
.clientId(clientId)
|
||||
.ackMode(AckMode.ALL)
|
||||
.batchSize(StorageUnit.zero)
|
||||
.keySerializer(UnKeyedSerde.serializer)
|
||||
.valueSerializer(ScalaSerdes.Thrift[ContextualizedFavoriteEvent].serializer)
|
||||
.build()
|
||||
}
|
||||
|
||||
private def getUUAConsumer(
|
||||
seekStrategy: SeekStrategy = SeekStrategy.BEGINNING,
|
||||
) = {
|
||||
val builder = FinagleKafkaConsumerBuilder()
|
||||
.dest(brokers.map(_.brokerList()).mkString(","))
|
||||
.clientId("consumer_uua")
|
||||
.groupId(KafkaGroupId("validator_uua"))
|
||||
.keyDeserializer(UnKeyedSerde.deserializer)
|
||||
.valueDeserializer(ScalaSerdes.Thrift[UnifiedUserAction].deserializer)
|
||||
.requestTimeout(Duration.fromSeconds(1))
|
||||
.enableAutoCommit(false)
|
||||
.seekStrategy(seekStrategy)
|
||||
|
||||
new KafkaConsumerClient(builder.config)
|
||||
}
|
||||
|
||||
test("TlsFavService starts") {
|
||||
server.assertHealthy()
|
||||
}
|
||||
|
||||
test("TlsFavService should process input events") {
|
||||
val producer = getProducer()
|
||||
val inputConsumer = getConsumer()
|
||||
val uuaConsumer = getUUAConsumer()
|
||||
|
||||
val favoriteEvent = FavoriteEventUnion.Favorite(FavoriteEvent(123L, 123L, 123L, 123L))
|
||||
val value =
|
||||
ContextualizedFavoriteEvent(favoriteEvent, LogEventContext("localhost", 123L))
|
||||
|
||||
try {
|
||||
server.assertHealthy()
|
||||
|
||||
// before, should be empty
|
||||
inputConsumer.subscribe(Set(KafkaTopic(inputTopic.topic)))
|
||||
assert(inputConsumer.poll().count() == 0)
|
||||
|
||||
// after, should contain at least a message
|
||||
await(producer.send(inputTopic.topic, new UnKeyed, value, System.currentTimeMillis))
|
||||
producer.flush()
|
||||
assert(inputConsumer.poll().count() == 1)
|
||||
|
||||
uuaConsumer.subscribe(Set(KafkaTopic(outputTopic.topic)))
|
||||
// This is tricky: it is not guaranteed that the TlsFavsService can process and output the
|
||||
// event to output topic faster than the below consumer. So we'd use a timer here which may
|
||||
// not be the best practice.
|
||||
// If someone finds the below test is flaky, please just remove the below test completely.
|
||||
Thread.sleep(5000L)
|
||||
assert(uuaConsumer.poll().count() == 1)
|
||||
} finally {
|
||||
await(producer.close())
|
||||
inputConsumer.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,50 @@
|
||||
package com.twitter.unified_user_actions.service
|
||||
|
||||
import com.twitter.inject.Test
|
||||
import com.twitter.kafka.client.headers.ATLA
|
||||
import com.twitter.kafka.client.headers.Implicits._
|
||||
import com.twitter.kafka.client.headers.PDXA
|
||||
import com.twitter.kafka.client.headers.Zone
|
||||
import com.twitter.unified_user_actions.service.module.ZoneFiltering
|
||||
import com.twitter.util.mock.Mockito
|
||||
import org.apache.kafka.clients.consumer.ConsumerRecord
|
||||
import org.junit.runner.RunWith
|
||||
import org.scalatestplus.junit.JUnitRunner
|
||||
import org.scalatest.prop.TableDrivenPropertyChecks
|
||||
|
||||
@RunWith(classOf[JUnitRunner])
|
||||
class ZoneFilteringTest extends Test with Mockito with TableDrivenPropertyChecks {
|
||||
trait Fixture {
|
||||
val consumerRecord =
|
||||
new ConsumerRecord[Array[Byte], Array[Byte]]("topic", 0, 0l, Array(0), Array(0))
|
||||
}
|
||||
|
||||
test("two DCs filter") {
|
||||
val zones = Table(
|
||||
"zone",
|
||||
Some(ATLA),
|
||||
Some(PDXA),
|
||||
None
|
||||
)
|
||||
forEvery(zones) { localZoneOpt: Option[Zone] =>
|
||||
forEvery(zones) { headerZoneOpt: Option[Zone] =>
|
||||
localZoneOpt.foreach { localZone =>
|
||||
new Fixture {
|
||||
headerZoneOpt match {
|
||||
case Some(headerZone) =>
|
||||
consumerRecord.headers().setZone(headerZone)
|
||||
if (headerZone == ATLA && localZone == ATLA)
|
||||
ZoneFiltering.localDCFiltering(consumerRecord, localZone) shouldBe true
|
||||
else if (headerZone == PDXA && localZone == PDXA)
|
||||
ZoneFiltering.localDCFiltering(consumerRecord, localZone) shouldBe true
|
||||
else
|
||||
ZoneFiltering.localDCFiltering(consumerRecord, localZone) shouldBe false
|
||||
case _ =>
|
||||
ZoneFiltering.localDCFiltering(consumerRecord, localZone) shouldBe true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user