INSANEMODE/the-algorithm-mirror
1
0
Fork 0
mirror of https://github.com/twitter/the-algorithm.git synced 2025-06-10 14:48:16 -05:00
Code Issues Packages Projects Releases Wiki Activity

[Medium][UUA] Clean up BCE in UUA

Browse Source 
This is to clean up the BCE adapters and services in UUA since BCE no longer exists.
This commit is contained in:
twitter-team
2023-04-14 15:28:51 -07:00
parent 4df87a278e
commit 23fa75d406
9 changed files with 0 additions and 569 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/BehavioralClientEventService.scala
View File

@ -1,25 +0,0 @@
package com.twitter.unified_user_actions.service;
import com.twitter.finatra.decider.modules.DeciderModule
import com.twitter.finatra.kafka.serde.UnKeyed
import com.twitter.inject.server.TwitterServer
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
import com.twitter.storage.behavioral_event.thriftscala.FlattenedEventLog
import com.twitter.unified_user_actions.service.module.KafkaProcessorBehavioralClientEventModule
object BehavioralClientEventServiceMain extends BehavioralClientEventService
class BehavioralClientEventService extends TwitterServer {
override val modules = Seq(
KafkaProcessorBehavioralClientEventModule,
DeciderModule
)
override protected def setup(): Unit = {}
override protected def start(): Unit = {
val processor = injector.instance[AtLeastOnceProcessor[UnKeyed, FlattenedEventLog]]
closeOnExit(processor)
processor.start()
}
}

87
unified_user_actions/service/src/main/scala/com/twitter/unified_user_actions/service/module/KafkaProcessorBehavioralClientEventModule.scala
View File

@ -1,87 +0,0 @@
package com.twitter.unified_user_actions.service.module
import com.google.inject.Provides
import com.twitter.decider.Decider
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.finatra.kafka.serde.UnKeyed
import com.twitter.finatra.kafka.serde.UnKeyedSerde
import com.twitter.inject.annotations.Flag
import com.twitter.inject.TwitterModule
import com.twitter.kafka.client.processor.AtLeastOnceProcessor
import com.twitter.storage.behavioral_event.thriftscala.FlattenedEventLog
import com.twitter.unified_user_actions.adapter.behavioral_client_event.BehavioralClientEventAdapter
import com.twitter.unified_user_actions.kafka.CompressionTypeFlag
import com.twitter.unified_user_actions.kafka.serde.NullableScalaSerdes
import com.twitter.util.Duration
import com.twitter.util.StorageUnit
import com.twitter.util.logging.Logging
import javax.inject.Singleton
object KafkaProcessorBehavioralClientEventModule extends TwitterModule with Logging {
override def modules = Seq(FlagsModule)
private val adapter: BehavioralClientEventAdapter = new BehavioralClientEventAdapter
private final val processorName: String = "uuaProcessor"
@Provides
@Singleton
def providesKafkaProcessor(
decider: Decider,
@Flag(FlagsModule.cluster) cluster: String,
@Flag(FlagsModule.kafkaSourceCluster) kafkaSourceCluster: String,
@Flag(FlagsModule.kafkaDestCluster) kafkaDestCluster: String,
@Flag(FlagsModule.kafkaSourceTopic) kafkaSourceTopic: String,
@Flag(FlagsModule.kafkaSinkTopics) kafkaSinkTopics: Seq[String],
@Flag(FlagsModule.kafkaGroupId) kafkaGroupId: String,
@Flag(FlagsModule.kafkaProducerClientId) kafkaProducerClientId: String,
@Flag(FlagsModule.kafkaMaxPendingRequests) kafkaMaxPendingRequests: Int,
@Flag(FlagsModule.kafkaWorkerThreads) kafkaWorkerThreads: Int,
@Flag(FlagsModule.commitInterval) commitInterval: Duration,
@Flag(FlagsModule.maxPollRecords) maxPollRecords: Int,
@Flag(FlagsModule.maxPollInterval) maxPollInterval: Duration,
@Flag(FlagsModule.sessionTimeout) sessionTimeout: Duration,
@Flag(FlagsModule.fetchMax) fetchMax: StorageUnit,
@Flag(FlagsModule.batchSize) batchSize: StorageUnit,
@Flag(FlagsModule.linger) linger: Duration,
@Flag(FlagsModule.bufferMem) bufferMem: StorageUnit,
@Flag(FlagsModule.compressionType) compressionTypeFlag: CompressionTypeFlag,
@Flag(FlagsModule.retries) retries: Int,
@Flag(FlagsModule.retryBackoff) retryBackoff: Duration,
@Flag(FlagsModule.requestTimeout) requestTimeout: Duration,
@Flag(FlagsModule.enableTrustStore) enableTrustStore: Boolean,
@Flag(FlagsModule.trustStoreLocation) trustStoreLocation: String,
statsReceiver: StatsReceiver,
): AtLeastOnceProcessor[UnKeyed, FlattenedEventLog] = {
KafkaProcessorProvider.provideDefaultAtLeastOnceProcessor(
name = processorName,
kafkaSourceCluster = kafkaSourceCluster,
kafkaGroupId = kafkaGroupId,
kafkaSourceTopic = kafkaSourceTopic,
sourceKeyDeserializer = UnKeyedSerde.deserializer,
sourceValueDeserializer = NullableScalaSerdes
.Thrift[FlattenedEventLog](statsReceiver.counter("deserializerErrors")).deserializer,
commitInterval = commitInterval,
maxPollRecords = maxPollRecords,
maxPollInterval = maxPollInterval,
sessionTimeout = sessionTimeout,
fetchMax = fetchMax,
processorMaxPendingRequests = kafkaMaxPendingRequests,
processorWorkerThreads = kafkaWorkerThreads,
adapter = adapter,
kafkaSinkTopics = kafkaSinkTopics,
kafkaDestCluster = kafkaDestCluster,
kafkaProducerClientId = kafkaProducerClientId,
batchSize = batchSize,
linger = linger,
bufferMem = bufferMem,
compressionType = compressionTypeFlag.compressionType,
retries = retries,
retryBackoff = retryBackoff,
requestTimeout = requestTimeout,
statsReceiver = statsReceiver,
trustStoreLocationOpt = if (enableTrustStore) Some(trustStoreLocation) else None,
decider = decider,
zone = ZoneFiltering.zoneMapping(cluster),
)
}
}