INSANEMODE/the-algorithm-mirror
1
0
Fork 0
mirror of https://github.com/twitter/the-algorithm.git synced 2025-06-12 15:48:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

Twitter Recommendation Algorithm

Browse Source 
Please note we have force-pushed a new initial commit in order to remove some publicly-available Twitter user information. Note that this process may be required in the future.
This commit is contained in:
twitter-team
2023-03-31 17:36:31 -05:00
commit ef4c5eb65e
5364 changed files with 460239 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
timelineranker/server/src/main/resources/BUILD.bazel Normal file
View File

@ -0,0 +1,5 @@
resources(
sources = [
"*.xml",
],
)

124
timelineranker/server/src/main/resources/logback-timelineranker.xml Normal file
View File

@ -0,0 +1,124 @@
<configuration>
<shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
<property name="async_queue_size" value="${queue.size:-50000}"/>
<property name="async_max_flush_time" value="${max.flush.time:-0}"/>
<property name="SERVICE_OUTPUT" value="${log.service.output:-server.log}"/>
<property name="DEBUG_TRANSCRIPTS_OUTPUT"
value="${log.debug_transcripts.output:-debug_transcripts.log}"/>
<property name="DEFAULT_SERVICE_PATTERN"
value="%5p [%d{yyyyMMdd-HH:mm:ss.SSS}] %logger{0}: %m%n"/>
<!-- JUL/JDK14 to Logback bridge -->
<contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
<resetJUL>true</resetJUL>
</contextListener>
<!-- Service Log -->
<appender name="SERVICE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${SERVICE_OUTPUT}</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<!-- daily rollover -->
<fileNamePattern>${SERVICE_OUTPUT}.%d.%i.gz</fileNamePattern>
<maxFileSize>500MB</maxFileSize>
<!-- keep 21 days' worth of history -->
<maxHistory>21</maxHistory>
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy>
<encoder>
<pattern>${DEFAULT_SERVICE_PATTERN}</pattern>
</encoder>
</appender>
<!-- debug transcripts -->
<appender name="DEBUG-TRANSCRIPTS" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${DEBUG_TRANSCRIPTS_OUTPUT}</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<!-- daily rollover -->
<fileNamePattern>${DEBUG_TRANSCRIPTS_OUTPUT}.%d.%i.gz</fileNamePattern>
<maxFileSize>500MB</maxFileSize>
<!-- keep 21 days' worth of history -->
<maxHistory>21</maxHistory>
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy>
<encoder>
<pattern>${DEFAULT_SERVICE_PATTERN}</pattern>
</encoder>
</appender>
<!-- LogLens/splunk -->
<appender name="LOGLENS" class="com.twitter.loglens.logback.LoglensAppender">
<mdcAdditionalContext>true</mdcAdditionalContext>
<category>loglens</category>
<index>${log.lens.index:-timelineranker}</index>
<tag>${log.lens.tag}</tag>
<encoder>
<pattern>%msg%n</pattern>
</encoder>
<filter class="com.twitter.strato.logging.logback.RegexFilter">
<forLogger>manhattan-client</forLogger>
<excludeRegex>.*InvalidRequest.*</excludeRegex>
</filter>
</appender>
<!-- ===================================================== -->
<!-- Primary Async Appenders -->
<!-- ===================================================== -->
<appender name="ASYNC-SERVICE" class="ch.qos.logback.classic.AsyncAppender">
<queueSize>${async_queue_size}</queueSize>
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
<appender-ref ref="SERVICE"/>
</appender>
<appender name="ASYNC-DEBUG-TRANSCRIPTS" class="ch.qos.logback.classic.AsyncAppender">
<queueSize>${async_queue_size}</queueSize>
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
<appender-ref ref="DEBUG-TRANSCRIPTS"/>
</appender>
<appender name="ASYNC-LOGLENS" class="ch.qos.logback.classic.AsyncAppender">
<queueSize>${async_queue_size}</queueSize>
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
<appender-ref ref="LOGLENS"/>
</appender>
<!-- ===================================================== -->
<!-- Package Config -->
<!-- ===================================================== -->
<!-- Per-Package Config -->
<logger name="OptimisticLockingCache" level="off"/>
<logger name="ZkSession" level="info"/>
<logger name="com.twitter" level="info"/>
<logger name="com.twitter.decider.StoreDecider" level="warn"/>
<logger name="com.twitter.distributedlog.client" level="warn"/>
<logger name="com.twitter.finagle.liveness" level="warn"/>
<logger name="com.twitter.finagle.mtls.authorization.config.AccessControlListConfiguration" level="warn"/>
<logger name="com.twitter.finagle.mux" level="warn"/>
<logger name="com.twitter.finagle.serverset2" level="warn"/>
<logger name="com.twitter.finatra.kafka.common.kerberoshelpers" level="warn"/>
<logger name="com.twitter.finatra.kafka.utils.BootstrapServerUtils" level="warn"/>
<logger name="com.twitter.logging.ScribeHandler" level="warn"/>
<logger name="com.twitter.server.coordinate" level="error"/>
<logger name="com.twitter.wilyns" level="warn"/>
<logger name="com.twitter.zookeeper.client" level="info"/>
<logger name="com.twitter.zookeeper.client.internal" level="warn"/>
<logger name="manhattan-client" level="warn"/>
<logger name="org.apache.kafka.clients.NetworkClient" level="error"/>
<logger name="org.apache.kafka.clients.consumer.internals" level="error"/>
<logger name="org.apache.kafka.clients.producer.internals" level="error"/>
<logger name="org.apache.kafka.common.network" level="warn"/>
<logger name="org.apache.zookeeper" level="error"/>
<logger name="org.apache.zookeeper.ClientCnxn" level="warn"/>
<!-- Root Config -->
<root level="${log_level:-INFO}">
<appender-ref ref="ASYNC-SERVICE"/>
<appender-ref ref="ASYNC-LOGLENS"/>
</root>
<!-- debug transcripts: logger name MUST be c.t.timelines.util.debuglog.DebugLog.DebugTranscriptsLog -->
<logger name="DebugTranscripts" level="info">
<appender-ref ref="ASYNC-DEBUG-TRANSCRIPTS"/>
<appender-ref ref="ASYNC-LOGLENS"/>
</logger>
</configuration>

32
timelineranker/server/src/main/scala/BUILD.bazel Normal file
View File

@ -0,0 +1,32 @@
target(
dependencies = [
"timelineranker/server/src/main/scala/com/twitter/timelineranker/repository",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/server",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/source",
],
)
jvm_binary(
name = "bin",
basename = "timelineranker-server",
main = "com.twitter.timelineranker.server.Main",
runtime_platform = "java11",
tags = ["bazel-compatible"],
dependencies = [
":scala",
"3rdparty/jvm/org/slf4j:jcl-over-slf4j", # [1]
"3rdparty/jvm/org/slf4j:log4j-over-slf4j", # [1]
"loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", # [2]
"strato/src/main/scala/com/twitter/strato/logging/logback", # [2]
"timelineranker/server/src/main/resources", # [2]
"twitter-server/logback-classic/src/main/scala", #[2]
],
)
# [1] bridge other logging implementations to slf4j-api in addition to JUL
# https://docbird.twitter.biz/core_libraries_guide/logging/twitter_server.html
# without these, c.t.l.Logger become silent/null logger since no proper
# configuration can be found. This can be removed once there are no
# depdency from service to c.t.l.Logger
#
# [2] incur logback implementation

22
timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/BUILD Normal file
View File

@ -0,0 +1,22 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"cortex-core/thrift/src/main/thrift:thrift-scala",
"cortex-tweet-annotate/service/src/main/thrift:thrift-scala",
"finagle/finagle-memcached/src/main/scala",
"mediaservices/commons/src/main/thrift:thrift-scala",
"servo/repo",
"servo/util/src/main/scala",
"src/thrift/com/twitter/ml/api:data-scala",
"src/thrift/com/twitter/ml/prediction_service:prediction_service-scala",
"timelines/src/main/scala/com/twitter/timelines/model/types",
"timelines/src/main/scala/com/twitter/timelines/util",
"timelines/src/main/scala/com/twitter/timelines/util/stats",
"util/util-core:util-core-util",
"util/util-logging/src/main/scala",
"util/util-stats/src/main/scala",
],
)

113
timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/CortexTweetQueryServiceClient.scala Normal file
View File

@ -0,0 +1,113 @@
package com.twitter.timelineranker.clients
import com.twitter.cortex_core.thriftscala.ModelName
import com.twitter.cortex_tweet_annotate.thriftscala._
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.logging.Logger
import com.twitter.mediaservices.commons.mediainformation.thriftscala.CalibrationLevel
import com.twitter.timelines.model.TweetId
import com.twitter.timelines.util.stats.RequestScope
import com.twitter.timelines.util.stats.RequestStats
import com.twitter.timelines.util.stats.ScopedFactory
import com.twitter.timelines.util.FailOpenHandler
import com.twitter.util.Future
object CortexTweetQueryServiceClient {
private[this] val logger = Logger.get(getClass.getSimpleName)
/**
* A tweet is considered safe if Cortex NSFA model gives it a score that is above the threshold.
* Both the score and the threshold are returned in a response from getTweetSignalByIds endpoint.
*/
private def getSafeTweet(
request: TweetSignalRequest,
response: ModelResponseResult
): Option[TweetId] = {
val tweetId = request.tweetId
response match {
case ModelResponseResult(ModelResponseState.Success, Some(tid), Some(modelResponse), _) =>
val prediction = modelResponse.predictions.flatMap(_.headOption)
val score = prediction.map(_.score.score)
val highRecallBucket = prediction.flatMap(_.calibrationBuckets).flatMap { buckets =>
buckets.find(_.description.contains(CalibrationLevel.HighRecall))
}
val threshold = highRecallBucket.map(_.threshold)
(score, threshold) match {
case (Some(s), Some(t)) if (s > t) =>
Some(tid)
case (Some(s), Some(t)) =>
logger.ifDebug(
s"Cortex NSFA score for tweet $tweetId is $s (threshold is $t), removing as unsafe."
)
None
case _ =>
logger.ifDebug(s"Unexpected response, removing tweet $tweetId as unsafe.")
None
}
case _ =>
logger.ifWarning(
s"Cortex tweet NSFA call was not successful, removing tweet $tweetId as unsafe."
)
None
}
}
}
/**
* Enables calling cortex tweet query service to get NSFA scores on the tweet.
*/
class CortexTweetQueryServiceClient(
cortexClient: CortexTweetQueryService.MethodPerEndpoint,
requestScope: RequestScope,
statsReceiver: StatsReceiver)
extends RequestStats {
import CortexTweetQueryServiceClient._
private[this] val logger = Logger.get(getClass.getSimpleName)
private[this] val getTweetSignalByIdsRequestStats =
requestScope.stats("cortex", statsReceiver, suffix = Some("getTweetSignalByIds"))
private[this] val getTweetSignalByIdsRequestScopedStatsReceiver =
getTweetSignalByIdsRequestStats.scopedStatsReceiver
private[this] val failedCortexTweetQueryServiceScope =
getTweetSignalByIdsRequestScopedStatsReceiver.scope(Failures)
private[this] val failedCortexTweetQueryServiceCallCounter =
failedCortexTweetQueryServiceScope.counter("failOpen")
private[this] val cortexTweetQueryServiceFailOpenHandler = new FailOpenHandler(
getTweetSignalByIdsRequestScopedStatsReceiver
)
def getSafeTweets(tweetIds: Seq[TweetId]): Future[Seq[TweetId]] = {
val requests = tweetIds.map { id => TweetSignalRequest(id, ModelName.TweetToNsfa) }
val results = cortexClient
.getTweetSignalByIds(
GetTweetSignalByIdsRequest(requests)
)
.map(_.results)
cortexTweetQueryServiceFailOpenHandler(
results.map { responses =>
requests.zip(responses).flatMap {
case (request, response) =>
getSafeTweet(request, response)
}
}
) { _ =>
failedCortexTweetQueryServiceCallCounter.incr()
logger.ifWarning(s"Cortex tweet NSFA call failed, considering tweets $tweetIds as unsafe.")
Future.value(Seq())
}
}
}
class ScopedCortexTweetQueryServiceClientFactory(
cortexClient: CortexTweetQueryService.MethodPerEndpoint,
statsReceiver: StatsReceiver)
extends ScopedFactory[CortexTweetQueryServiceClient] {
override def scope(scope: RequestScope): CortexTweetQueryServiceClient = {
new CortexTweetQueryServiceClient(cortexClient, scope, statsReceiver)
}
}

48
timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/MemcacheFactory.scala Normal file
View File

@ -0,0 +1,48 @@
package com.twitter.timelineranker.clients
import com.twitter.finagle.memcached.{Client => FinagleMemcacheClient}
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.logging.Logger
import com.twitter.servo.cache.FinagleMemcache
import com.twitter.servo.cache.MemcacheCache
import com.twitter.servo.cache.ObservableMemcache
import com.twitter.servo.cache.Serializer
import com.twitter.servo.cache.StatsReceiverCacheObserver
import com.twitter.timelines.util.stats.RequestScope
import com.twitter.timelines.util.stats.ScopedFactory
import com.twitter.util.Duration
/**
* Factory to create a servo Memcache-backed Cache object. Clients are required to provide a
* serializer/deserializer for keys and values.
*/
class MemcacheFactory(memcacheClient: FinagleMemcacheClient, statsReceiver: StatsReceiver) {
private[this] val logger = Logger.get(getClass.getSimpleName)
def apply[K, V](
keySerializer: K => String,
valueSerializer: Serializer[V],
ttl: Duration
): MemcacheCache[K, V] = {
new MemcacheCache[K, V](
memcache = new ObservableMemcache(
new FinagleMemcache(memcacheClient),
new StatsReceiverCacheObserver(statsReceiver, 1000, logger)
),
ttl = ttl,
serializer = valueSerializer,
transformKey = keySerializer
)
}
}
class ScopedMemcacheFactory(memcacheClient: FinagleMemcacheClient, statsReceiver: StatsReceiver)
extends ScopedFactory[MemcacheFactory] {
override def scope(scope: RequestScope): MemcacheFactory = {
new MemcacheFactory(
memcacheClient,
statsReceiver.scope("memcache", scope.scope)
)
}
}

24
timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/content_features_cache/BUILD Normal file
View File

@ -0,0 +1,24 @@
scala_library(
sources = ["*.scala"],
platform = "java8",
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"3rdparty/jvm/com/twitter/algebird:bijection",
"3rdparty/jvm/com/twitter/bijection:core",
"3rdparty/jvm/com/twitter/bijection:netty",
"3rdparty/jvm/com/twitter/bijection:scrooge",
"3rdparty/jvm/com/twitter/bijection:thrift",
"3rdparty/jvm/com/twitter/bijection:util",
"3rdparty/jvm/com/twitter/storehaus:core",
"finagle/finagle-stats",
"scrooge/scrooge-core/src/main/scala",
"src/scala/com/twitter/summingbird_internal/bijection:bijection-implicits",
"src/thrift/com/twitter/timelines/content_features:thrift-scala",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model",
"timelines/src/main/scala/com/twitter/timelines/clients/memcache_common",
"timelines/src/main/scala/com/twitter/timelines/model/types",
"util/util-core:util-core-util",
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
],
)

39
timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/content_features_cache/ContentFeaturesMemcacheBuilder.scala Normal file
View File

@ -0,0 +1,39 @@
package com.twitter.timelineranker.clients.content_features_cache
import com.twitter.bijection.Injection
import com.twitter.bijection.scrooge.CompactScalaCodec
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.storehaus.Store
import com.twitter.timelineranker.recap.model.ContentFeatures
import com.twitter.timelines.clients.memcache_common._
import com.twitter.timelines.content_features.{thriftscala => thrift}
import com.twitter.timelines.model.TweetId
import com.twitter.util.Duration
/**
* Content features will be stored by tweetId
*/
class ContentFeaturesMemcacheBuilder(
config: StorehausMemcacheConfig,
ttl: Duration,
statsReceiver: StatsReceiver) {
private[this] val scalaToThriftInjection: Injection[ContentFeatures, thrift.ContentFeatures] =
Injection.build[ContentFeatures, thrift.ContentFeatures](_.toThrift)(
ContentFeatures.tryFromThrift)
private[this] val thriftToBytesInjection: Injection[thrift.ContentFeatures, Array[Byte]] =
CompactScalaCodec(thrift.ContentFeatures)
private[this] implicit val valueInjection: Injection[ContentFeatures, Array[Byte]] =
scalaToThriftInjection.andThen(thriftToBytesInjection)
private[this] val underlyingBuilder =
new MemcacheStoreBuilder[TweetId, ContentFeatures](
config = config,
scopeName = "contentFeaturesCache",
statsReceiver = statsReceiver,
ttl = ttl
)
def build(): Store[TweetId, ContentFeatures] = underlyingBuilder.build()
}

43
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/BUILD Normal file
View File

@ -0,0 +1,43 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"3rdparty/jvm/com/twitter/storehaus:core",
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
"finagle/finagle-core/src/main",
"servo/util/src/main/scala",
"src/thrift/com/twitter/search:earlybird-scala",
"src/thrift/com/twitter/search/common:constants-scala",
"src/thrift/com/twitter/search/common:features-scala",
"src/thrift/com/twitter/service/metastore/gen:thrift-scala",
"timelineranker/common/src/main/scala/com/twitter/timelineranker/model",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/contentfeatures",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/core",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/util",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/visibility",
"timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck",
"timelines/src/main/scala/com/twitter/timelines/clients/manhattan",
"timelines/src/main/scala/com/twitter/timelines/clients/relevance_search",
"timelines/src/main/scala/com/twitter/timelines/clients/tweetypie",
"timelines/src/main/scala/com/twitter/timelines/common/model",
"timelines/src/main/scala/com/twitter/timelines/earlybird/common/options",
"timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils",
"timelines/src/main/scala/com/twitter/timelines/model/candidate",
"timelines/src/main/scala/com/twitter/timelines/model/tweet",
"timelines/src/main/scala/com/twitter/timelines/util",
"timelines/src/main/scala/com/twitter/timelines/util/bounds",
"timelines/src/main/scala/com/twitter/timelines/util/stats",
"timelines/src/main/scala/com/twitter/timelines/visibility",
"timelines/src/main/scala/com/twitter/timelines/visibility/model",
"util/util-core:util-core-util",
"util/util-core/src/main/scala/com/twitter/conversions",
"util/util-logging/src/main/scala/com/twitter/logging",
"util/util-stats/src/main/scala",
],
)

40
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/CandidateGenerationTransform.scala Normal file
View File

@ -0,0 +1,40 @@
package com.twitter.timelineranker.common
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope
import com.twitter.timelineranker.model.CandidateTweet
import com.twitter.timelineranker.model.CandidateTweetsResult
import com.twitter.util.Future
class CandidateGenerationTransform(statsReceiver: StatsReceiver)
extends FutureArrow[HydratedCandidatesAndFeaturesEnvelope, CandidateTweetsResult] {
private[this] val numCandidateTweetsStat = statsReceiver.stat("numCandidateTweets")
private[this] val numSourceTweetsStat = statsReceiver.stat("numSourceTweets")
override def apply(
candidatesAndFeaturesEnvelope: HydratedCandidatesAndFeaturesEnvelope
): Future[CandidateTweetsResult] = {
val hydratedTweets = candidatesAndFeaturesEnvelope.candidateEnvelope.hydratedTweets.outerTweets
if (hydratedTweets.nonEmpty) {
val candidates = hydratedTweets.map { hydratedTweet =>
CandidateTweet(hydratedTweet, candidatesAndFeaturesEnvelope.features(hydratedTweet.tweetId))
}
numCandidateTweetsStat.add(candidates.size)
val sourceTweets =
candidatesAndFeaturesEnvelope.candidateEnvelope.sourceHydratedTweets.outerTweets.map {
hydratedTweet =>
CandidateTweet(
hydratedTweet,
candidatesAndFeaturesEnvelope.features(hydratedTweet.tweetId))
}
numSourceTweetsStat.add(sourceTweets.size)
Future.value(CandidateTweetsResult(candidates, sourceTweets))
} else {
Future.value(CandidateTweetsResult.Empty)
}
}
}

112
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/ContentFeaturesHydrationTransform.scala Normal file
View File

@ -0,0 +1,112 @@
package com.twitter.timelineranker.common
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.servo.util.FutureArrow
import com.twitter.servo.util.Gate
import com.twitter.storehaus.Store
import com.twitter.timelineranker.contentfeatures.ContentFeaturesProvider
import com.twitter.timelineranker.core.FutureDependencyTransformer
import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope
import com.twitter.timelineranker.model.RecapQuery
import com.twitter.timelineranker.recap.model.ContentFeatures
import com.twitter.timelineranker.util.SearchResultUtil._
import com.twitter.timelineranker.util.CachingContentFeaturesProvider
import com.twitter.timelineranker.util.TweetHydrator
import com.twitter.timelineranker.util.TweetypieContentFeaturesProvider
import com.twitter.timelines.clients.tweetypie.TweetyPieClient
import com.twitter.timelines.model.TweetId
import com.twitter.util.Future
import com.twitter.timelines.configapi
import com.twitter.timelines.util.FutureUtils
class ContentFeaturesHydrationTransformBuilder(
tweetyPieClient: TweetyPieClient,
contentFeaturesCache: Store[TweetId, ContentFeatures],
enableContentFeaturesGate: Gate[RecapQuery],
enableTokensInContentFeaturesGate: Gate[RecapQuery],
enableTweetTextInContentFeaturesGate: Gate[RecapQuery],
enableConversationControlContentFeaturesGate: Gate[RecapQuery],
enableTweetMediaHydrationGate: Gate[RecapQuery],
hydrateInReplyToTweets: Boolean,
statsReceiver: StatsReceiver) {
val scopedStatsReceiver: StatsReceiver = statsReceiver.scope("ContentFeaturesHydrationTransform")
val tweetHydrator: TweetHydrator = new TweetHydrator(tweetyPieClient, scopedStatsReceiver)
val tweetypieContentFeaturesProvider: ContentFeaturesProvider =
new TweetypieContentFeaturesProvider(
tweetHydrator,
enableContentFeaturesGate,
enableTokensInContentFeaturesGate,
enableTweetTextInContentFeaturesGate,
enableConversationControlContentFeaturesGate,
enableTweetMediaHydrationGate,
scopedStatsReceiver
)
val cachingContentFeaturesProvider: ContentFeaturesProvider = new CachingContentFeaturesProvider(
underlying = tweetypieContentFeaturesProvider,
contentFeaturesCache = contentFeaturesCache,
statsReceiver = scopedStatsReceiver
)
val contentFeaturesProvider: configapi.FutureDependencyTransformer[RecapQuery, Seq[TweetId], Map[
TweetId,
ContentFeatures
]] = FutureDependencyTransformer.partition(
gate = enableContentFeaturesGate,
ifTrue = cachingContentFeaturesProvider,
ifFalse = tweetypieContentFeaturesProvider
)
lazy val contentFeaturesHydrationTransform: ContentFeaturesHydrationTransform =
new ContentFeaturesHydrationTransform(
contentFeaturesProvider,
enableContentFeaturesGate,
hydrateInReplyToTweets
)
def build(): ContentFeaturesHydrationTransform = contentFeaturesHydrationTransform
}
class ContentFeaturesHydrationTransform(
contentFeaturesProvider: ContentFeaturesProvider,
enableContentFeaturesGate: Gate[RecapQuery],
hydrateInReplyToTweets: Boolean)
extends FutureArrow[
HydratedCandidatesAndFeaturesEnvelope,
HydratedCandidatesAndFeaturesEnvelope
] {
override def apply(
request: HydratedCandidatesAndFeaturesEnvelope
): Future[HydratedCandidatesAndFeaturesEnvelope] = {
if (enableContentFeaturesGate(request.candidateEnvelope.query)) {
val searchResults = request.candidateEnvelope.searchResults
val sourceTweetIdMap = searchResults.map { searchResult =>
(searchResult.id, getRetweetSourceTweetId(searchResult).getOrElse(searchResult.id))
}.toMap
val inReplyToTweetIds = if (hydrateInReplyToTweets) {
searchResults.flatMap(getInReplyToTweetId)
} else {
Seq.empty
}
val tweetIdsToHydrate = (sourceTweetIdMap.values ++ inReplyToTweetIds).toSeq.distinct
val contentFeaturesMapFuture = if (tweetIdsToHydrate.nonEmpty) {
contentFeaturesProvider(request.candidateEnvelope.query, tweetIdsToHydrate)
} else {
FutureUtils.EmptyMap[TweetId, ContentFeatures]
}
Future.value(
request.copy(
contentFeaturesFuture = contentFeaturesMapFuture,
tweetSourceTweetMap = sourceTweetIdMap,
inReplyToTweetIds = inReplyToTweetIds.toSet
)
)
} else {
Future.value(request)
}
}
}

15
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/CreateCandidateEnvelopeTransform.scala Normal file
View File

@ -0,0 +1,15 @@
package com.twitter.timelineranker.common
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.core.CandidateEnvelope
import com.twitter.timelineranker.model.RecapQuery
import com.twitter.util.Future
/**
* Create a CandidateEnvelope based on the incoming RecapQuery
*/
object CreateCandidateEnvelopeTransform extends FutureArrow[RecapQuery, CandidateEnvelope] {
override def apply(query: RecapQuery): Future[CandidateEnvelope] = {
Future.value(CandidateEnvelope(query))
}
}

33
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/FeatureHydrationDataTransform.scala Normal file
View File

@ -0,0 +1,33 @@
package com.twitter.timelineranker.common
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.core.CandidateEnvelope
import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope
import com.twitter.timelineranker.model.RecapQuery
import com.twitter.util.Future
/**
* Fetches all data required for feature hydration and generates the HydratedCandidatesAndFeaturesEnvelope
* @param tweetHydrationAndFilteringPipeline Pipeline which fetches the candidate tweets, hydrates and filters them
* @param languagesService Fetch user languages, required for feature hydration
* @param userProfileInfoService Fetch user profile info, required for feature hydration
*/
class FeatureHydrationDataTransform(
tweetHydrationAndFilteringPipeline: FutureArrow[RecapQuery, CandidateEnvelope],
languagesService: UserLanguagesTransform,
userProfileInfoService: UserProfileInfoTransform)
extends FutureArrow[RecapQuery, HydratedCandidatesAndFeaturesEnvelope] {
override def apply(request: RecapQuery): Future[HydratedCandidatesAndFeaturesEnvelope] = {
Future
.join(
languagesService(request),
userProfileInfoService(request),
tweetHydrationAndFilteringPipeline(request)).map {
case (languages, userProfileInfo, transformedCandidateEnvelope) =>
HydratedCandidatesAndFeaturesEnvelope(
transformedCandidateEnvelope,
languages,
userProfileInfo)
}
}
}

198
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/FollowAndRealGraphCombiningTransform.scala Normal file
View File

@ -0,0 +1,198 @@
package com.twitter.timelineranker.common
import com.twitter.finagle.stats.Stat
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.core.CandidateEnvelope
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
import com.twitter.timelineranker.parameters.recap.RecapQueryContext
import com.twitter.timelineranker.parameters.in_network_tweets.InNetworkTweetParams.RecycledMaxFollowedUsersEnableAntiDilutionParam
import com.twitter.timelineranker.visibility.FollowGraphDataProvider
import com.twitter.timelines.earlybird.common.options.AuthorScoreAdjustments
import com.twitter.util.Future
/**
* Transform which conditionally augments follow graph data using the real graph,
* derived from the earlybirdOptions passed in the query
*
* @param followGraphDataProvider data provider to be used for fetching updated mutual follow info
* @param maxFollowedUsersProvider max number of users to return
* @param enableRealGraphUsersProvider should we augment using real graph data?
* @param maxRealGraphAndFollowedUsersProvider max combined users to return, overrides maxFollowedUsersProvider above
* @param statsReceiver scoped stats received
*/
class FollowAndRealGraphCombiningTransform(
followGraphDataProvider: FollowGraphDataProvider,
maxFollowedUsersProvider: DependencyProvider[Int],
enableRealGraphUsersProvider: DependencyProvider[Boolean],
maxRealGraphAndFollowedUsersProvider: DependencyProvider[Int],
imputeRealGraphAuthorWeightsProvider: DependencyProvider[Boolean],
imputeRealGraphAuthorWeightsPercentileProvider: DependencyProvider[Int],
statsReceiver: StatsReceiver)
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
// Number of authors in the seedset after mixing followed users and real graph users
// Only have this stat if user follows >= maxFollowedUsers and enableRealGraphUsers is true and onlyRealGraphUsers is false
val numFollowAndRealGraphUsersStat: Stat = statsReceiver.stat("numFollowAndRealGraphUsers")
val numFollowAndRealGraphUsersFromFollowGraphStat =
statsReceiver.scope("numFollowAndRealGraphUsers").stat("FollowGraphUsers")
val numFollowAndRealGraphUsersFromRealGraphStat =
statsReceiver.scope("numFollowAndRealGraphUsers").stat("RealGraphUsers")
val numFollowAndRealGraphUsersFromRealGraphCounter =
statsReceiver.scope("numFollowAndRealGraphUsers").counter()
// Number of authors in the seedset with only followed users
// Only have this stat if user follows >= maxFollowedUsers and enableRealGraphUsers is false
val numFollowedUsersStat: Stat = statsReceiver.stat("numFollowedUsers")
// Number of authors in the seedset with only followed users
// Only have this stat if user follows < maxFollowedUsers
val numFollowedUsersLessThanMaxStat: Stat = statsReceiver.stat("numFollowedUsersLessThanMax")
val numFollowedUsersLessThanMaxCounter =
statsReceiver.scope("numFollowedUsersLessThanMax").counter()
val numFollowedUsersMoreThanMaxStat: Stat = statsReceiver.stat("numFollowedUsersMoreThanMax")
val numFollowedUsersMoreThanMaxCounter =
statsReceiver.scope("numFollowedUsersMoreThanMax").counter()
val realGraphAuthorWeightsSumProdStat: Stat = statsReceiver.stat("realGraphAuthorWeightsSumProd")
val realGraphAuthorWeightsSumMinExpStat: Stat =
statsReceiver.stat("realGraphAuthorWeightsSumMinExp")
val realGraphAuthorWeightsSumP50ExpStat: Stat =
statsReceiver.stat("realGraphAuthorWeightsSumP50Exp")
val realGraphAuthorWeightsSumP95ExpStat: Stat =
statsReceiver.stat("realGraphAuthorWeightsSumP95Exp")
val numAuthorsWithoutRealgraphScoreStat: Stat =
statsReceiver.stat("numAuthorsWithoutRealgraphScore")
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
val realGraphData = envelope.query.earlybirdOptions
.map(_.authorScoreAdjustments.authorScoreMap)
.getOrElse(Map.empty)
Future
.join(
envelope.followGraphData.followedUserIdsFuture,
envelope.followGraphData.mutedUserIdsFuture
).map {
case (followedUserIds, mutedUserIds) =>
// Anti-dilution param for DDG-16198
val recycledMaxFollowedUsersEnableAntiDilutionParamProvider =
DependencyProvider.from(RecycledMaxFollowedUsersEnableAntiDilutionParam)
val maxFollowedUsers = {
if (followedUserIds.size > RecapQueryContext.MaxFollowedUsers.default && recycledMaxFollowedUsersEnableAntiDilutionParamProvider(
envelope.query)) {
// trigger experiment
maxFollowedUsersProvider(envelope.query)
} else {
maxFollowedUsersProvider(envelope.query)
}
}
val filteredRealGraphUserIds = realGraphData.keySet
.filterNot(mutedUserIds)
.take(maxFollowedUsers)
.toSeq
val filteredFollowedUserIds = followedUserIds.filterNot(mutedUserIds)
if (followedUserIds.size < maxFollowedUsers) {
numFollowedUsersLessThanMaxStat.add(filteredFollowedUserIds.size)
// stats
numFollowedUsersLessThanMaxCounter.incr()
(filteredFollowedUserIds, false)
} else {
numFollowedUsersMoreThanMaxStat.add(filteredFollowedUserIds.size)
numFollowedUsersMoreThanMaxCounter.incr()
if (enableRealGraphUsersProvider(envelope.query)) {
val maxRealGraphAndFollowedUsersNum =
maxRealGraphAndFollowedUsersProvider(envelope.query)
require(
maxRealGraphAndFollowedUsersNum >= maxFollowedUsers,
"maxRealGraphAndFollowedUsers must be greater than or equal to maxFollowedUsers."
)
val recentFollowedUsersNum = RecapQueryContext.MaxFollowedUsers.bounds
.apply(maxRealGraphAndFollowedUsersNum - filteredRealGraphUserIds.size)
val recentFollowedUsers =
filteredFollowedUserIds
.filterNot(filteredRealGraphUserIds.contains)
.take(recentFollowedUsersNum)
val filteredFollowAndRealGraphUserIds =
recentFollowedUsers ++ filteredRealGraphUserIds
// Track the size of recentFollowedUsers from SGS
numFollowAndRealGraphUsersFromFollowGraphStat.add(recentFollowedUsers.size)
// Track the size of filteredRealGraphUserIds from real graph dataset.
numFollowAndRealGraphUsersFromRealGraphStat.add(filteredRealGraphUserIds.size)
numFollowAndRealGraphUsersFromRealGraphCounter.incr()
numFollowAndRealGraphUsersStat.add(filteredFollowAndRealGraphUserIds.size)
(filteredFollowAndRealGraphUserIds, true)
} else {
numFollowedUsersStat.add(followedUserIds.size)
(filteredFollowedUserIds, false)
}
}
}.map {
case (updatedFollowSeq, shouldUpdateMutualFollows) =>
val updatedMutualFollowing = if (shouldUpdateMutualFollows) {
followGraphDataProvider.getMutuallyFollowingUserIds(
envelope.query.userId,
updatedFollowSeq)
} else {
envelope.followGraphData.mutuallyFollowingUserIdsFuture
}
val followGraphData = envelope.followGraphData.copy(
followedUserIdsFuture = Future.value(updatedFollowSeq),
mutuallyFollowingUserIdsFuture = updatedMutualFollowing
)
val authorIdsWithRealgraphScore = realGraphData.keySet
val authorIdsWithoutRealgraphScores =
updatedFollowSeq.filterNot(authorIdsWithRealgraphScore.contains)
//stat for logging the percentage of users' followings that do not have a realgraph score
if (updatedFollowSeq.nonEmpty)
numAuthorsWithoutRealgraphScoreStat.add(
authorIdsWithoutRealgraphScores.size / updatedFollowSeq.size * 100)
if (imputeRealGraphAuthorWeightsProvider(envelope.query) && realGraphData.nonEmpty) {
val imputedScorePercentile =
imputeRealGraphAuthorWeightsPercentileProvider(envelope.query) / 100.0
val existingAuthorIdScores = realGraphData.values.toList.sorted
val imputedScoreIndex = Math.min(
existingAuthorIdScores.length - 1,
(existingAuthorIdScores.length * imputedScorePercentile).toInt)
val imputedScore = existingAuthorIdScores(imputedScoreIndex)
val updatedAuthorScoreMap = realGraphData ++ authorIdsWithoutRealgraphScores
.map(_ -> imputedScore).toMap
imputedScorePercentile match {
case 0.0 =>
realGraphAuthorWeightsSumMinExpStat.add(updatedAuthorScoreMap.values.sum.toFloat)
case 0.5 =>
realGraphAuthorWeightsSumP50ExpStat.add(updatedAuthorScoreMap.values.sum.toFloat)
case 0.95 =>
realGraphAuthorWeightsSumP95ExpStat.add(updatedAuthorScoreMap.values.sum.toFloat)
case _ =>
}
val earlybirdOptionsWithUpdatedAuthorScoreMap = envelope.query.earlybirdOptions
.map(_.copy(authorScoreAdjustments = AuthorScoreAdjustments(updatedAuthorScoreMap)))
val updatedQuery =
envelope.query.copy(earlybirdOptions = earlybirdOptionsWithUpdatedAuthorScoreMap)
envelope.copy(query = updatedQuery, followGraphData = followGraphData)
} else {
envelope.query.earlybirdOptions
.map(_.authorScoreAdjustments.authorScoreMap.values.sum.toFloat).foreach {
realGraphAuthorWeightsSumProdStat.add(_)
}
envelope.copy(followGraphData = followGraphData)
}
}
}
}

23
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/FollowGraphDataTransform.scala Normal file
View File

@ -0,0 +1,23 @@
package com.twitter.timelineranker.common
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.core.CandidateEnvelope
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
import com.twitter.timelineranker.visibility.FollowGraphDataProvider
import com.twitter.util.Future
class FollowGraphDataTransform(
followGraphDataProvider: FollowGraphDataProvider,
maxFollowedUsersProvider: DependencyProvider[Int])
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
val followGraphData = followGraphDataProvider.getAsync(
envelope.query.userId,
maxFollowedUsersProvider(envelope.query)
)
Future.value(envelope.copy(followGraphData = followGraphData))
}
}

31
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/HydrateTweetsAndSourceTweetsInParallelTransform.scala Normal file
View File

@ -0,0 +1,31 @@
package com.twitter.timelineranker.common
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.core.CandidateEnvelope
import com.twitter.util.Future
/**
* Transform that explicitly hydrates candidate tweets and fetches source tweets in parallel
* and then joins the results back into the original Envelope
* @param candidateTweetHydration Pipeline that hydrates candidate tweets
* @param sourceTweetHydration Pipeline that fetches and hydrates source tweets
*/
class HydrateTweetsAndSourceTweetsInParallelTransform(
candidateTweetHydration: FutureArrow[CandidateEnvelope, CandidateEnvelope],
sourceTweetHydration: FutureArrow[CandidateEnvelope, CandidateEnvelope])
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
Future
.join(
candidateTweetHydration(envelope),
sourceTweetHydration(envelope)
).map {
case (candidateTweetEnvelope, sourceTweetEnvelope) =>
envelope.copy(
hydratedTweets = candidateTweetEnvelope.hydratedTweets,
sourceSearchResults = sourceTweetEnvelope.sourceSearchResults,
sourceHydratedTweets = sourceTweetEnvelope.sourceHydratedTweets
)
}
}
}

96
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/HydratedTweetsFilterTransform.scala Normal file
View File

@ -0,0 +1,96 @@
package com.twitter.timelineranker.common
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.logging.Logger
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.core.CandidateEnvelope
import com.twitter.timelineranker.core.HydratedTweets
import com.twitter.timelineranker.util.TweetFilters
import com.twitter.timelineranker.util.TweetsPostFilter
import com.twitter.timelines.model.UserId
import com.twitter.util.Future
object HydratedTweetsFilterTransform {
val EmptyFollowGraphDataTuple: (Seq[UserId], Seq[UserId], Set[UserId]) =
(Seq.empty[UserId], Seq.empty[UserId], Set.empty[UserId])
val DefaultNumRetweetsAllowed = 1
// Number of duplicate retweets (including the first one) allowed.
// For example,
// If there are 7 retweets of a given tweet, the following value will cause 5 of them
// to be returned after filtering and the additional 2 will be filtered out.
val NumDuplicateRetweetsAllowed = 5
}
/**
* Transform which takes TweetFilters ValueSets for inner and outer tweets and uses
* TweetsPostFilter to filter down the HydratedTweets using the supplied filters
*
* @param useFollowGraphData - use follow graph for filtering; otherwise only does filtering
* independent of follow graph data
* @param useSourceTweets - only needed when filtering extended replies
* @param statsReceiver - scoped stats receiver
*/
class HydratedTweetsFilterTransform(
outerFilters: TweetFilters.ValueSet,
innerFilters: TweetFilters.ValueSet,
useFollowGraphData: Boolean,
useSourceTweets: Boolean,
statsReceiver: StatsReceiver,
numRetweetsAllowed: Int = HydratedTweetsFilterTransform.DefaultNumRetweetsAllowed)
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
import HydratedTweetsFilterTransform._
val logger: Logger = Logger.get(getClass.getSimpleName)
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
if (outerFilters == TweetFilters.None) {
Future.value(envelope)
} else {
val tweetsPostOuterFilter = new TweetsPostFilter(outerFilters, logger, statsReceiver)
val tweetsPostInnerFilter = new TweetsPostFilter(innerFilters, logger, statsReceiver)
val graphData = if (useFollowGraphData) {
Future.join(
envelope.followGraphData.followedUserIdsFuture,
envelope.followGraphData.inNetworkUserIdsFuture,
envelope.followGraphData.mutedUserIdsFuture
)
} else {
Future.value(EmptyFollowGraphDataTuple)
}
val sourceTweets = if (useSourceTweets) {
envelope.sourceHydratedTweets.outerTweets
} else {
Nil
}
graphData.map {
case (followedUserIds, inNetworkUserIds, mutedUserIds) =>
val outerTweets = tweetsPostOuterFilter(
userId = envelope.query.userId,
followedUserIds = followedUserIds,
inNetworkUserIds = inNetworkUserIds,
mutedUserIds = mutedUserIds,
tweets = envelope.hydratedTweets.outerTweets,
numRetweetsAllowed = numRetweetsAllowed,
sourceTweets = sourceTweets
)
val innerTweets = tweetsPostInnerFilter(
userId = envelope.query.userId,
followedUserIds = followedUserIds,
inNetworkUserIds = inNetworkUserIds,
mutedUserIds = mutedUserIds,
// inner tweets refers to quoted tweets not source tweets, and special rulesets
// in birdherd handle visibility of viewer to inner tweet author for these tweets.
tweets = envelope.hydratedTweets.innerTweets,
numRetweetsAllowed = numRetweetsAllowed,
sourceTweets = sourceTweets
)
envelope.copy(hydratedTweets = HydratedTweets(outerTweets, innerTweets))
}
}
}
}

38
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/InNetworkTweetsSearchFeaturesHydrationTransform.scala Normal file
View File

@ -0,0 +1,38 @@
package com.twitter.timelineranker.common
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope
import com.twitter.timelines.earlybird.common.utils.EarlybirdFeaturesHydrator
import com.twitter.util.Future
object InNetworkTweetsSearchFeaturesHydrationTransform
extends FutureArrow[
HydratedCandidatesAndFeaturesEnvelope,
HydratedCandidatesAndFeaturesEnvelope
] {
override def apply(
request: HydratedCandidatesAndFeaturesEnvelope
): Future[HydratedCandidatesAndFeaturesEnvelope] = {
Future
.join(
request.candidateEnvelope.followGraphData.followedUserIdsFuture,
request.candidateEnvelope.followGraphData.mutuallyFollowingUserIdsFuture
).map {
case (followedIds, mutuallyFollowingIds) =>
val featuresByTweetId = EarlybirdFeaturesHydrator.hydrate(
searcherUserId = request.candidateEnvelope.query.userId,
searcherProfileInfo = request.userProfileInfo,
followedUserIds = followedIds,
mutuallyFollowingUserIds = mutuallyFollowingIds,
userLanguages = request.userLanguages,
uiLanguageCode = request.candidateEnvelope.query.deviceContext.flatMap(_.languageCode),
searchResults = request.candidateEnvelope.searchResults,
sourceTweetSearchResults = request.candidateEnvelope.sourceSearchResults,
tweets = request.candidateEnvelope.hydratedTweets.outerTweets,
sourceTweets = request.candidateEnvelope.sourceHydratedTweets.outerTweets
)
request.copy(features = featuresByTweetId)
}
}
}

55
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/MarkRandomTweetTransform.scala Normal file
View File

@ -0,0 +1,55 @@
package com.twitter.timelineranker.common
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.core.CandidateEnvelope
import com.twitter.timelineranker.model.CandidateTweet
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
import com.twitter.util.Future
import com.twitter.util.Time
import scala.util.Random
/**
* picks up one or more random tweets and sets its tweetFeatures.isRandomTweet field to true.
*/
class MarkRandomTweetTransform(
includeRandomTweetProvider: DependencyProvider[Boolean],
randomGenerator: Random = new Random(Time.now.inMilliseconds),
includeSingleRandomTweetProvider: DependencyProvider[Boolean],
probabilityRandomTweetProvider: DependencyProvider[Double])
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
val includeRandomTweet = includeRandomTweetProvider(envelope.query)
val includeSingleRandomTweet = includeSingleRandomTweetProvider(envelope.query)
val probabilityRandomTweet = probabilityRandomTweetProvider(envelope.query)
val searchResults = envelope.searchResults
if (!includeRandomTweet || searchResults.isEmpty) { // random tweet off
Future.value(envelope)
} else if (includeSingleRandomTweet) { // pick only one
val randomIdx = randomGenerator.nextInt(searchResults.size)
val randomTweet = searchResults(randomIdx)
val randomTweetWithFlag = randomTweet.copy(
tweetFeatures = randomTweet.tweetFeatures
.orElse(Some(CandidateTweet.DefaultFeatures))
.map(_.copy(isRandomTweet = Some(true)))
)
val updatedSearchResults = searchResults.updated(randomIdx, randomTweetWithFlag)
Future.value(envelope.copy(searchResults = updatedSearchResults))
} else { // pick tweets with perTweetProbability
val updatedSearchResults = searchResults.map { result =>
if (randomGenerator.nextDouble() < probabilityRandomTweet) {
result.copy(
tweetFeatures = result.tweetFeatures
.orElse(Some(CandidateTweet.DefaultFeatures))
.map(_.copy(isRandomTweet = Some(true))))
} else
result
}
Future.value(envelope.copy(searchResults = updatedSearchResults))
}
}
}

49
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/OutOfNetworkRepliesToUserIdSearchResultsTransform.scala Normal file
View File

@ -0,0 +1,49 @@
package com.twitter.timelineranker.common
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.core.CandidateEnvelope
import com.twitter.timelines.clients.relevance_search.SearchClient
import com.twitter.util.Future
object OutOfNetworkRepliesToUserIdSearchResultsTransform {
val DefaultMaxTweetCount = 100
}
// Requests search results for out-of-network replies to a user Id
class OutOfNetworkRepliesToUserIdSearchResultsTransform(
searchClient: SearchClient,
statsReceiver: StatsReceiver,
logSearchDebugInfo: Boolean = true)
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
private[this] val maxCountStat = statsReceiver.stat("maxCount")
private[this] val numResultsFromSearchStat = statsReceiver.stat("numResultsFromSearch")
private[this] val earlybirdScoreX100Stat = statsReceiver.stat("earlybirdScoreX100")
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
val maxCount = envelope.query.maxCount
.getOrElse(OutOfNetworkRepliesToUserIdSearchResultsTransform.DefaultMaxTweetCount)
maxCountStat.add(maxCount)
envelope.followGraphData.followedUserIdsFuture
.flatMap {
case followedIds =>
searchClient
.getOutOfNetworkRepliesToUserId(
userId = envelope.query.userId,
followedUserIds = followedIds.toSet,
maxCount = maxCount,
earlybirdOptions = envelope.query.earlybirdOptions,
logSearchDebugInfo
).map { results =>
numResultsFromSearchStat.add(results.size)
results.foreach { result =>
val earlybirdScoreX100 =
result.metadata.flatMap(_.score).getOrElse(0.0).toFloat * 100
earlybirdScoreX100Stat.add(earlybirdScoreX100)
}
envelope.copy(searchResults = results)
}
}
}
}

31
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/OutOfNetworkTweetsSearchFeaturesHydrationTransform.scala Normal file
View File

@ -0,0 +1,31 @@
package com.twitter.timelineranker.common
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope
import com.twitter.timelines.earlybird.common.utils.EarlybirdFeaturesHydrator
import com.twitter.util.Future
object OutOfNetworkTweetsSearchFeaturesHydrationTransform
extends FutureArrow[
HydratedCandidatesAndFeaturesEnvelope,
HydratedCandidatesAndFeaturesEnvelope
] {
override def apply(
request: HydratedCandidatesAndFeaturesEnvelope
): Future[HydratedCandidatesAndFeaturesEnvelope] = {
val featuresByTweetId = EarlybirdFeaturesHydrator.hydrate(
searcherUserId = request.candidateEnvelope.query.userId,
searcherProfileInfo = request.userProfileInfo,
followedUserIds = Seq.empty,
mutuallyFollowingUserIds = Set.empty,
userLanguages = request.userLanguages,
uiLanguageCode = request.candidateEnvelope.query.deviceContext.flatMap(_.languageCode),
searchResults = request.candidateEnvelope.searchResults,
sourceTweetSearchResults = Seq.empty,
tweets = request.candidateEnvelope.hydratedTweets.outerTweets,
sourceTweets = Seq.empty
)
Future.value(request.copy(features = featuresByTweetId))
}
}

29
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/RecapHydrationSearchResultsTransformBase.scala Normal file
View File

@ -0,0 +1,29 @@
package com.twitter.timelineranker.common
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.core.CandidateEnvelope
import com.twitter.timelines.clients.relevance_search.SearchClient
import com.twitter.timelines.model.TweetId
import com.twitter.util.Future
trait RecapHydrationSearchResultsTransformBase
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
protected def statsReceiver: StatsReceiver
protected def searchClient: SearchClient
private[this] val numResultsFromSearchStat = statsReceiver.stat("numResultsFromSearch")
def tweetIdsToHydrate(envelope: CandidateEnvelope): Seq[TweetId]
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
searchClient
.getTweetsScoredForRecap(
envelope.query.userId,
tweetIdsToHydrate(envelope),
envelope.query.earlybirdOptions
).map { results =>
numResultsFromSearchStat.add(results.size)
envelope.copy(searchResults = results)
}
}
}

89
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/RecapSearchResultsTransform.scala Normal file
View File

@ -0,0 +1,89 @@
package com.twitter.timelineranker.common
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.core.CandidateEnvelope
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
import com.twitter.timelineranker.model.TweetIdRange
import com.twitter.timelineranker.parameters.recap.RecapParams
import com.twitter.timelines.clients.relevance_search.SearchClient
import com.twitter.timelines.clients.relevance_search.SearchClient.TweetTypes
import com.twitter.util.Future
/**
* Fetch recap/recycled search results using the search client
* and populate them into the CandidateEnvelope
*/
class RecapSearchResultsTransform(
searchClient: SearchClient,
maxCountProvider: DependencyProvider[Int],
returnAllResultsProvider: DependencyProvider[Boolean],
relevanceOptionsMaxHitsToProcessProvider: DependencyProvider[Int],
enableExcludeSourceTweetIdsProvider: DependencyProvider[Boolean],
enableSettingTweetTypesWithTweetKindOptionProvider: DependencyProvider[Boolean],
perRequestSearchClientIdProvider: DependencyProvider[Option[String]],
relevanceSearchProvider: DependencyProvider[Boolean] =
DependencyProvider.from(RecapParams.EnableRelevanceSearchParam),
statsReceiver: StatsReceiver,
logSearchDebugInfo: Boolean = true)
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
private[this] val maxCountStat = statsReceiver.stat("maxCount")
private[this] val numResultsFromSearchStat = statsReceiver.stat("numResultsFromSearch")
private[this] val excludedTweetIdsStat = statsReceiver.stat("excludedTweetIds")
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
val maxCount = maxCountProvider(envelope.query)
maxCountStat.add(maxCount)
val excludedTweetIdsOpt = envelope.query.excludedTweetIds
excludedTweetIdsOpt.foreach { excludedTweetIds =>
excludedTweetIdsStat.add(excludedTweetIds.size)
}
val tweetIdRange = envelope.query.range
.map(TweetIdRange.fromTimelineRange)
.getOrElse(TweetIdRange.default)
val beforeTweetIdExclusive = tweetIdRange.toId
val afterTweetIdExclusive = tweetIdRange.fromId
val returnAllResults = returnAllResultsProvider(envelope.query)
val relevanceOptionsMaxHitsToProcess = relevanceOptionsMaxHitsToProcessProvider(envelope.query)
Future
.join(
envelope.followGraphData.followedUserIdsFuture,
envelope.followGraphData.retweetsMutedUserIdsFuture
).flatMap {
case (followedIds, retweetsMutedIds) =>
val followedIdsIncludingSelf = followedIds.toSet + envelope.query.userId
searchClient
.getUsersTweetsForRecap(
userId = envelope.query.userId,
followedUserIds = followedIdsIncludingSelf,
retweetsMutedUserIds = retweetsMutedIds,
maxCount = maxCount,
tweetTypes = TweetTypes.fromTweetKindOption(envelope.query.options),
searchOperator = envelope.query.searchOperator,
beforeTweetIdExclusive = beforeTweetIdExclusive,
afterTweetIdExclusive = afterTweetIdExclusive,
enableSettingTweetTypesWithTweetKindOption =
enableSettingTweetTypesWithTweetKindOptionProvider(envelope.query),
excludedTweetIds = excludedTweetIdsOpt,
earlybirdOptions = envelope.query.earlybirdOptions,
getOnlyProtectedTweets = false,
logSearchDebugInfo = logSearchDebugInfo,
returnAllResults = returnAllResults,
enableExcludeSourceTweetIdsQuery =
enableExcludeSourceTweetIdsProvider(envelope.query),
relevanceSearch = relevanceSearchProvider(envelope.query),
searchClientId = perRequestSearchClientIdProvider(envelope.query),
relevanceOptionsMaxHitsToProcess = relevanceOptionsMaxHitsToProcess
).map { results =>
numResultsFromSearchStat.add(results.size)
envelope.copy(searchResults = results)
}
}
}
}

67
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/RecapSearchResultsTruncationTransform.scala Normal file
View File

@ -0,0 +1,67 @@
package com.twitter.timelineranker.common
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.core.CandidateEnvelope
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
import com.twitter.timelineranker.util.SearchResultUtil
import com.twitter.util.Future
/**
* Truncate the search results by score. Assumes that the search results are sorted in
* score-descending order unless extraSortBeforeTruncation is set to true.
*
* This transform has two main use cases:
*
* - when returnAllResults is set to true, earlybird returns (numResultsPerShard * number of shards)
* results. this transform is then used to further truncate the result, so that the size will be the
* same as when returnAllResults is set to false.
*
* - we retrieve extra number of results from earlybird, as specified in MaxCountMultiplierParam,
* so that we are left with sufficient number of candidates after hydration and filtering.
* this transform will be used to get rid of extra results we ended up not using.
*/
class RecapSearchResultsTruncationTransform(
extraSortBeforeTruncationGate: DependencyProvider[Boolean],
maxCountProvider: DependencyProvider[Int],
statsReceiver: StatsReceiver)
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
private[this] val postTruncationSizeStat = statsReceiver.stat("postTruncationSize")
private[this] val earlybirdScoreX100Stat = statsReceiver.stat("earlybirdScoreX100")
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
val sortBeforeTruncation = extraSortBeforeTruncationGate(envelope.query)
val maxCount = maxCountProvider(envelope.query)
val searchResults = envelope.searchResults
// set aside results that are marked by isRandomTweet field
val (randomTweetSeq, searchResultsExcludingRandom) = searchResults.partition { result =>
result.tweetFeatures.flatMap(_.isRandomTweet).getOrElse(false)
}
// sort and truncate searchResults other than the random tweet
val maxCountExcludingRandom = Math.max(0, maxCount - randomTweetSeq.size)
val truncatedResultsExcludingRandom =
if (sortBeforeTruncation || searchResultsExcludingRandom.size > maxCountExcludingRandom) {
val sorted = if (sortBeforeTruncation) {
searchResultsExcludingRandom.sortWith(
SearchResultUtil.getScore(_) > SearchResultUtil.getScore(_))
} else searchResultsExcludingRandom
sorted.take(maxCountExcludingRandom)
} else searchResultsExcludingRandom
// put back the random tweet set aside previously
val allTruncatedResults = truncatedResultsExcludingRandom ++ randomTweetSeq
// stats
postTruncationSizeStat.add(allTruncatedResults.size)
allTruncatedResults.foreach { result =>
val earlybirdScoreX100 =
result.metadata.flatMap(_.score).getOrElse(0.0).toFloat * 100
earlybirdScoreX100Stat.add(earlybirdScoreX100)
}
Future.value(envelope.copy(searchResults = allTruncatedResults))
}
}

23
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/SearchResultDedupAndSortingTransform.scala Normal file
View File

@ -0,0 +1,23 @@
package com.twitter.timelineranker.common
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.core.CandidateEnvelope
import com.twitter.timelines.model.TweetId
import com.twitter.util.Future
import scala.collection.mutable
/**
* Remove duplicate search results and order them reverse-chron.
*/
object SearchResultDedupAndSortingTransform
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
val seenTweetIds = mutable.Set.empty[TweetId]
val dedupedResults = envelope.searchResults
.filter(result => seenTweetIds.add(result.id))
.sortBy(_.id)(Ordering[TweetId].reverse)
val transformedEnvelope = envelope.copy(searchResults = dedupedResults)
Future.value(transformedEnvelope)
}
}

62
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/SourceTweetsSearchResultsTransform.scala Normal file
View File

@ -0,0 +1,62 @@
package com.twitter.timelineranker.common
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.search.earlybird.thriftscala.ThriftSearchResult
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.core.CandidateEnvelope
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
import com.twitter.timelineranker.util.SourceTweetsUtil
import com.twitter.timelines.clients.relevance_search.SearchClient
import com.twitter.timelines.util.FailOpenHandler
import com.twitter.util.Future
object SourceTweetsSearchResultsTransform {
val EmptySearchResults: Seq[ThriftSearchResult] = Seq.empty[ThriftSearchResult]
val EmptySearchResultsFuture: Future[Seq[ThriftSearchResult]] = Future.value(EmptySearchResults)
}
/**
* Fetch source tweets for a given set of search results
* Collects ids of source tweets, including extended reply and reply source tweets if needed,
* fetches those tweets from search and populates them into the envelope
*/
class SourceTweetsSearchResultsTransform(
searchClient: SearchClient,
failOpenHandler: FailOpenHandler,
hydrateReplyRootTweetProvider: DependencyProvider[Boolean],
perRequestSourceSearchClientIdProvider: DependencyProvider[Option[String]],
statsReceiver: StatsReceiver)
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
import SourceTweetsSearchResultsTransform._
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
failOpenHandler {
envelope.followGraphData.followedUserIdsFuture.flatMap { followedUserIds =>
// NOTE: tweetIds are pre-computed as a performance optimisation.
val searchResultsTweetIds = envelope.searchResults.map(_.id).toSet
val sourceTweetIds = SourceTweetsUtil.getSourceTweetIds(
searchResults = envelope.searchResults,
searchResultsTweetIds = searchResultsTweetIds,
followedUserIds = followedUserIds,
shouldIncludeReplyRootTweets = hydrateReplyRootTweetProvider(envelope.query),
statsReceiver = scopedStatsReceiver
)
if (sourceTweetIds.isEmpty) {
EmptySearchResultsFuture
} else {
searchClient.getTweetsScoredForRecap(
userId = envelope.query.userId,
tweetIds = sourceTweetIds,
earlybirdOptions = envelope.query.earlybirdOptions,
logSearchDebugInfo = false,
searchClientId = perRequestSourceSearchClientIdProvider(envelope.query)
)
}
}
} { _: Throwable => EmptySearchResultsFuture }.map { sourceSearchResults =>
envelope.copy(sourceSearchResults = sourceSearchResults)
}
}
}

37
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TrimToMatchHydratedTweetsTransform.scala Normal file
View File

@ -0,0 +1,37 @@
package com.twitter.timelineranker.common
import com.twitter.search.earlybird.thriftscala.ThriftSearchResult
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.core.CandidateEnvelope
import com.twitter.timelines.model.tweet.HydratedTweet
import com.twitter.util.Future
/**
* trims searchResults to match with hydratedTweets
* (if we previously filtered out hydrated tweets, this transform filters the search result set
* down to match the hydrated tweets.)
*/
object TrimToMatchHydratedTweetsTransform
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
val filteredSearchResults =
trimSearchResults(envelope.searchResults, envelope.hydratedTweets.outerTweets)
val filteredSourceSearchResults =
trimSearchResults(envelope.sourceSearchResults, envelope.sourceHydratedTweets.outerTweets)
Future.value(
envelope.copy(
searchResults = filteredSearchResults,
sourceSearchResults = filteredSourceSearchResults
)
)
}
private def trimSearchResults(
searchResults: Seq[ThriftSearchResult],
hydratedTweets: Seq[HydratedTweet]
): Seq[ThriftSearchResult] = {
val filteredTweetIds = hydratedTweets.map(_.tweetId).toSet
searchResults.filter(result => filteredTweetIds.contains(result.id))
}
}

57
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TrimToMatchSearchResultsTransform.scala Normal file
View File

@ -0,0 +1,57 @@
package com.twitter.timelineranker.common
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.core.CandidateEnvelope
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
import com.twitter.timelineranker.util.SourceTweetsUtil
import com.twitter.util.Future
/**
* trims elements of the envelope other than the searchResults
* (i.e. sourceSearchResults, hydratedTweets, sourceHydratedTweets) to match with searchResults.
*/
class TrimToMatchSearchResultsTransform(
hydrateReplyRootTweetProvider: DependencyProvider[Boolean],
statsReceiver: StatsReceiver)
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
val searchResults = envelope.searchResults
val searchResultsIds = searchResults.map(_.id).toSet
// Trim rest of the seqs to match top search results.
val hydratedTweets = envelope.hydratedTweets.outerTweets
val topHydratedTweets = hydratedTweets.filter(ht => searchResultsIds.contains(ht.tweetId))
envelope.followGraphData.followedUserIdsFuture.map { followedUserIds =>
val sourceTweetIdsOfTopResults =
SourceTweetsUtil
.getSourceTweetIds(
searchResults = searchResults,
searchResultsTweetIds = searchResultsIds,
followedUserIds = followedUserIds,
shouldIncludeReplyRootTweets = hydrateReplyRootTweetProvider(envelope.query),
statsReceiver = scopedStatsReceiver
).toSet
val sourceTweetSearchResultsForTopN =
envelope.sourceSearchResults.filter(r => sourceTweetIdsOfTopResults.contains(r.id))
val hydratedSourceTweetsForTopN =
envelope.sourceHydratedTweets.outerTweets.filter(ht =>
sourceTweetIdsOfTopResults.contains(ht.tweetId))
val hydratedTweetsForEnvelope = envelope.hydratedTweets.copy(outerTweets = topHydratedTweets)
val hydratedSourceTweetsForEnvelope =
envelope.sourceHydratedTweets.copy(outerTweets = hydratedSourceTweetsForTopN)
envelope.copy(
hydratedTweets = hydratedTweetsForEnvelope,
searchResults = searchResults,
sourceHydratedTweets = hydratedSourceTweetsForEnvelope,
sourceSearchResults = sourceTweetSearchResultsForTopN
)
}
}
}

62
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TweetHydrationTransform.scala Normal file
View File

@ -0,0 +1,62 @@
package com.twitter.timelineranker.common
import com.twitter.conversions.DurationOps._
import com.twitter.finagle.IndividualRequestTimeoutException
import com.twitter.search.earlybird.thriftscala.ThriftSearchResult
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.core.CandidateEnvelope
import com.twitter.timelineranker.core.HydratedTweets
import com.twitter.timelineranker.model.PartiallyHydratedTweet
import com.twitter.timelines.model.tweet.HydratedTweet
import com.twitter.util.Future
object TweetHydrationTransform {
val EmptyHydratedTweets: HydratedTweets =
HydratedTweets(Seq.empty[HydratedTweet], Seq.empty[HydratedTweet])
val EmptyHydratedTweetsFuture: Future[HydratedTweets] = Future.value(EmptyHydratedTweets)
}
object CandidateTweetHydrationTransform extends TweetHydrationTransform {
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
hydrate(
searchResults = envelope.searchResults,
envelope = envelope
).map { tweets => envelope.copy(hydratedTweets = tweets) }
}
}
object SourceTweetHydrationTransform extends TweetHydrationTransform {
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
hydrate(
searchResults = envelope.sourceSearchResults,
envelope = envelope
).map { tweets => envelope.copy(sourceHydratedTweets = tweets) }
}
}
// Static IRTE to indicate timeout in tweet hydrator. Placeholder timeout duration of 0 millis is used
// since we are only concerned with the source of the exception.
object TweetHydrationTimeoutException extends IndividualRequestTimeoutException(0.millis) {
serviceName = "tweetHydrator"
}
/**
* Transform which hydrates tweets in the CandidateEnvelope
**/
trait TweetHydrationTransform extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
import TweetHydrationTransform._
protected def hydrate(
searchResults: Seq[ThriftSearchResult],
envelope: CandidateEnvelope
): Future[HydratedTweets] = {
if (searchResults.nonEmpty) {
Future.value(
HydratedTweets(searchResults.map(PartiallyHydratedTweet.fromSearchResult))
)
} else {
EmptyHydratedTweetsFuture
}
}
}

85
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/TweetKindOptionHydratedTweetsFilterTransform.scala Normal file
View File

@ -0,0 +1,85 @@
package com.twitter.timelineranker.common
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.servo.util.FutureArrow
import com.twitter.servo.util.Gate
import com.twitter.timelineranker.core.CandidateEnvelope
import com.twitter.timelineranker.model.RecapQuery
import com.twitter.timelineranker.parameters.recap.RecapParams
import com.twitter.timelineranker.parameters.uteg_liked_by_tweets.UtegLikedByTweetsParams
import com.twitter.timelineranker.util.TweetFilters
import com.twitter.timelines.common.model.TweetKindOption
import com.twitter.util.Future
import scala.collection.mutable
object TweetKindOptionHydratedTweetsFilterTransform {
private[common] val enableExpandedExtendedRepliesGate: Gate[RecapQuery] =
RecapQuery.paramGate(RecapParams.EnableExpandedExtendedRepliesFilterParam)
private[common] val excludeRecommendedRepliesToNonFollowedUsersGate: Gate[RecapQuery] =
RecapQuery.paramGate(
UtegLikedByTweetsParams.UTEGRecommendationsFilter.ExcludeRecommendedRepliesToNonFollowedUsersParam)
}
/**
* Filter hydrated tweets dynamically based on TweetKindOptions in the query.
*/
class TweetKindOptionHydratedTweetsFilterTransform(
useFollowGraphData: Boolean,
useSourceTweets: Boolean,
statsReceiver: StatsReceiver)
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
import TweetKindOptionHydratedTweetsFilterTransform._
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
val filters = convertToFilters(envelope)
val filterTransform = if (filters == TweetFilters.ValueSet.empty) {
FutureArrow.identity[CandidateEnvelope]
} else {
new HydratedTweetsFilterTransform(
outerFilters = filters,
innerFilters = TweetFilters.None,
useFollowGraphData = useFollowGraphData,
useSourceTweets = useSourceTweets,
statsReceiver = statsReceiver,
numRetweetsAllowed = HydratedTweetsFilterTransform.NumDuplicateRetweetsAllowed
)
}
filterTransform.apply(envelope)
}
/**
* Converts the given query options to equivalent TweetFilter values.
*
* Note:
* -- The semantic of TweetKindOption is opposite of that of TweetFilters.
* TweetKindOption values are of the form IncludeX. That is, they result in X being added.
* TweetFilters values specify what to exclude.
* -- IncludeExtendedReplies requires IncludeReplies to be also specified to be effective.
*/
private[common] def convertToFilters(envelope: CandidateEnvelope): TweetFilters.ValueSet = {
val queryOptions = envelope.query.options
val filters = mutable.Set.empty[TweetFilters.Value]
if (queryOptions.contains(TweetKindOption.IncludeReplies)) {
if (excludeRecommendedRepliesToNonFollowedUsersGate(
envelope.query) && envelope.query.utegLikedByTweetsOptions.isDefined) {
filters += TweetFilters.RecommendedRepliesToNotFollowedUsers
} else if (queryOptions.contains(TweetKindOption.IncludeExtendedReplies)) {
if (enableExpandedExtendedRepliesGate(envelope.query)) {
filters += TweetFilters.NotValidExpandedExtendedReplies
} else {
filters += TweetFilters.NotQualifiedExtendedReplies
}
} else {
filters += TweetFilters.ExtendedReplies
}
} else {
filters += TweetFilters.Replies
}
if (!queryOptions.contains(TweetKindOption.IncludeRetweets)) {
filters += TweetFilters.Retweets
}
TweetFilters.ValueSet.empty ++ filters
}
}

30
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/UserLanguagesTransform.scala Normal file
View File

@ -0,0 +1,30 @@
package com.twitter.timelineranker.common
import com.twitter.search.common.constants.thriftscala.ThriftLanguage
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.model.RecapQuery
import com.twitter.timelines.clients.manhattan.LanguageUtils
import com.twitter.timelines.clients.manhattan.UserMetadataClient
import com.twitter.timelines.util.FailOpenHandler
import com.twitter.util.Future
import com.twitter.service.metastore.gen.thriftscala.UserLanguages
object UserLanguagesTransform {
val EmptyUserLanguagesFuture: Future[UserLanguages] =
Future.value(UserMetadataClient.EmptyUserLanguages)
}
/**
* FutureArrow which fetches user languages
* It should be run in parallel with the main pipeline which fetches and hydrates CandidateTweets
*/
class UserLanguagesTransform(handler: FailOpenHandler, userMetadataClient: UserMetadataClient)
extends FutureArrow[RecapQuery, Seq[ThriftLanguage]] {
override def apply(request: RecapQuery): Future[Seq[ThriftLanguage]] = {
import UserLanguagesTransform._
handler {
userMetadataClient.getUserLanguages(request.userId)
} { _: Throwable => EmptyUserLanguagesFuture }
}.map(LanguageUtils.computeLanguages(_))
}

29
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/UserProfileInfoTransform.scala Normal file
View File

@ -0,0 +1,29 @@
package com.twitter.timelineranker.common
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.model.RecapQuery
import com.twitter.timelines.clients.gizmoduck.GizmoduckClient
import com.twitter.timelines.clients.gizmoduck.UserProfileInfo
import com.twitter.timelines.util.FailOpenHandler
import com.twitter.util.Future
object UserProfileInfoTransform {
val EmptyUserProfileInfo: UserProfileInfo = UserProfileInfo(None, None, None, None)
val EmptyUserProfileInfoFuture: Future[UserProfileInfo] = Future.value(EmptyUserProfileInfo)
}
/**
* FutureArrow which fetches user profile info
* It should be run in parallel with the main pipeline which fetches and hydrates CandidateTweets
*/
class UserProfileInfoTransform(handler: FailOpenHandler, gizmoduckClient: GizmoduckClient)
extends FutureArrow[RecapQuery, UserProfileInfo] {
import UserProfileInfoTransform._
override def apply(request: RecapQuery): Future[UserProfileInfo] = {
handler {
gizmoduckClient.getProfileInfo(request.userId).map { profileInfoOpt =>
profileInfoOpt.getOrElse(EmptyUserProfileInfo)
}
} { _: Throwable => EmptyUserProfileInfoFuture }
}
}

22
timelineranker/server/src/main/scala/com/twitter/timelineranker/common/VisibilityEnforcingTransform.scala Normal file
View File

@ -0,0 +1,22 @@
package com.twitter.timelineranker.common
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.core.CandidateEnvelope
import com.twitter.timelineranker.core.HydratedTweets
import com.twitter.timelines.visibility.VisibilityEnforcer
import com.twitter.util.Future
/**
* Transform which uses an instance of a VisiblityEnforcer to filter down HydratedTweets
*/
class VisibilityEnforcingTransform(visibilityEnforcer: VisibilityEnforcer)
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
visibilityEnforcer.apply(Some(envelope.query.userId), envelope.hydratedTweets.outerTweets).map {
visibleTweets =>
val innerTweets = envelope.hydratedTweets.innerTweets
envelope.copy(
hydratedTweets = HydratedTweets(outerTweets = visibleTweets, innerTweets = innerTweets))
}
}
}

65
timelineranker/server/src/main/scala/com/twitter/timelineranker/config/BUILD Normal file
View File

@ -0,0 +1,65 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"3rdparty/jvm/com/twitter/storehaus:core",
"3rdparty/jvm/org/apache/thrift:libthrift",
"abdecider",
"cortex-tweet-annotate/service/src/main/thrift:thrift-scala",
"decider",
"featureswitches/featureswitches-core/src/main/scala",
"finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication",
"finagle/finagle-core/src/main",
"finagle/finagle-memcached/src/main/scala",
"finagle/finagle-stats",
"finagle/finagle-thrift/src/main/java",
"finagle/finagle-thrift/src/main/scala",
"finagle/finagle-thriftmux/src/main/scala",
"loglens/loglens-logging/src/main/scala",
"merlin/util/src/main/scala",
"servo/decider",
"servo/request/src/main/scala",
"src/thrift/com/twitter/gizmoduck:thrift-scala",
"src/thrift/com/twitter/manhattan:v1-scala",
"src/thrift/com/twitter/merlin:thrift-scala",
"src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
"src/thrift/com/twitter/search:earlybird-scala",
"src/thrift/com/twitter/socialgraph:thrift-scala",
"src/thrift/com/twitter/timelineranker:thrift-scala",
"src/thrift/com/twitter/timelineservice:thrift-scala",
"src/thrift/com/twitter/tweetypie:service-scala",
"src/thrift/com/twitter/tweetypie:tweet-scala",
"strato/src/main/scala/com/twitter/strato/client",
"timelineranker/server/config",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/clients",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/content_features_cache",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/decider",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model",
"timelines:authorization",
"timelines:config",
"timelines:features",
"timelines:util",
"timelines:visibility",
"timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck",
"timelines/src/main/scala/com/twitter/timelines/clients/manhattan",
"timelines/src/main/scala/com/twitter/timelines/clients/memcache_common",
"timelines/src/main/scala/com/twitter/timelines/clients/socialgraph",
"timelines/src/main/scala/com/twitter/timelines/clients/strato/realgraph",
"timelines/src/main/scala/com/twitter/timelines/clients/tweetypie",
"timelines/src/main/scala/com/twitter/timelines/config/configapi",
"timelines/src/main/scala/com/twitter/timelines/util",
"timelines/src/main/scala/com/twitter/timelines/util/logging",
"timelines/src/main/scala/com/twitter/timelines/util/stats",
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
"twitter-server-internal",
"util/util-app",
"util/util-core:util-core-util",
"util/util-core/src/main/scala/com/twitter/conversions",
"util/util-stats/src/main/scala",
],
exports = [
"src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
],
)

119
timelineranker/server/src/main/scala/com/twitter/timelineranker/config/CallInfo.scala Normal file
View File

@ -0,0 +1,119 @@
package com.twitter.timelineranker.config
import com.twitter.conversions.DurationOps._
import com.twitter.util.Duration
import java.util.concurrent.TimeUnit
/**
* Information about a single method call.
*
* The purpose of this class is to allow one to express a call graph and latency associated with each (sub)call.
* Once a call graph is defined, calling getOverAllLatency() off the top level call returns total time taken by that call.
* That value can then be compared with the timeout budget allocated to that call to see if the
* value fits within the overall timeout budget of that call.
*
* This is useful in case of a complex call graph where it is hard to mentally estimate the effect on
* overall latency when updating timeout value of one or more sub-calls.
*
* @param methodName name of the called method.
* @param latency P999 Latency incurred in calling a service if the method calls an external service. Otherwise 0.
* @param dependsOn Other calls that this call depends on.
*/
case class Call(
methodName: String,
latency: Duration = 0.milliseconds,
dependsOn: Seq[Call] = Nil) {
/**
* Latency incurred in this call as well as recursively all calls this call depends on.
*/
def getOverAllLatency: Duration = {
val dependencyLatency = if (dependsOn.isEmpty) {
0.milliseconds
} else {
dependsOn.map(_.getOverAllLatency).max
}
latency + dependencyLatency
}
/**
* Call paths starting at this call and recursively traversing all dependencies.
* Typically used for debugging or logging.
*/
def getLatencyPaths: String = {
val sb = new StringBuilder
getLatencyPaths(sb, 1)
sb.toString
}
def getLatencyPaths(sb: StringBuilder, level: Int): Unit = {
sb.append(s"${getPrefix(level)} ${getLatencyString(getOverAllLatency)} $methodName\n")
if ((latency > 0.milliseconds) && !dependsOn.isEmpty) {
sb.append(s"${getPrefix(level + 1)} ${getLatencyString(latency)} self\n")
}
dependsOn.foreach(_.getLatencyPaths(sb, level + 1))
}
private def getLatencyString(latencyValue: Duration): String = {
val latencyMs = latencyValue.inUnit(TimeUnit.MILLISECONDS)
f"[$latencyMs%3d]"
}
private def getPrefix(level: Int): String = {
" " * (level * 4) + "--"
}
}
/**
* Information about the getRecapTweetCandidates call.
*
* Acronyms used:
* : call internal to TLR
* EB : Earlybird (search super root)
* GZ : Gizmoduck
* MH : Manhattan
* SGS : Social graph service
*
* The latency values are based on p9999 values observed over 1 week.
*/
object GetRecycledTweetCandidatesCall {
val getUserProfileInfo: Call = Call("GZ.getUserProfileInfo", 200.milliseconds)
val getUserLanguages: Call = Call("MH.getUserLanguages", 300.milliseconds) // p99: 15ms
val getFollowing: Call = Call("SGS.getFollowing", 250.milliseconds) // p99: 75ms
val getMutuallyFollowing: Call =
Call("SGS.getMutuallyFollowing", 400.milliseconds, Seq(getFollowing)) // p99: 100
val getVisibilityProfiles: Call =
Call("SGS.getVisibilityProfiles", 400.milliseconds, Seq(getFollowing)) // p99: 100
val getVisibilityData: Call = Call(
"getVisibilityData",
dependsOn = Seq(getFollowing, getMutuallyFollowing, getVisibilityProfiles)
)
val getTweetsForRecapRegular: Call =
Call("EB.getTweetsForRecap(regular)", 500.milliseconds, Seq(getVisibilityData)) // p99: 250
val getTweetsForRecapProtected: Call =
Call("EB.getTweetsForRecap(protected)", 250.milliseconds, Seq(getVisibilityData)) // p99: 150
val getSearchResults: Call =
Call("getSearchResults", dependsOn = Seq(getTweetsForRecapRegular, getTweetsForRecapProtected))
val getTweetsScoredForRecap: Call =
Call("EB.getTweetsScoredForRecap", 400.milliseconds, Seq(getSearchResults)) // p99: 100
val hydrateSearchResults: Call = Call("hydrateSearchResults")
val getSourceTweetSearchResults: Call =
Call("getSourceTweetSearchResults", dependsOn = Seq(getSearchResults))
val hydrateTweets: Call =
Call("hydrateTweets", dependsOn = Seq(getSearchResults, hydrateSearchResults))
val hydrateSourceTweets: Call =
Call("hydrateSourceTweets", dependsOn = Seq(getSourceTweetSearchResults, hydrateSearchResults))
val topLevel: Call = Call(
"getRecapTweetCandidates",
dependsOn = Seq(
getUserProfileInfo,
getUserLanguages,
getVisibilityData,
getSearchResults,
hydrateSearchResults,
hydrateSourceTweets
)
)
}

287
timelineranker/server/src/main/scala/com/twitter/timelineranker/config/ClientAccessPermissions.scala Normal file
View File

@ -0,0 +1,287 @@
package com.twitter.timelineranker.config
import com.twitter.timelineranker.decider.DeciderKey._
import com.twitter.timelines.authorization.TrustedPermission
import com.twitter.timelines.authorization.RateLimitingTrustedPermission
import com.twitter.timelines.authorization.RateLimitingUntrustedPermission
import com.twitter.timelines.authorization.ClientDetails
object ClientAccessPermissions {
// We want timelineranker locked down for requests outside of what's defined here.
val DefaultRateLimit = 0d
def unknown(name: String): ClientDetails = {
ClientDetails(name, RateLimitingUntrustedPermission(RateLimitOverrideUnknown, DefaultRateLimit))
}
val All: Seq[ClientDetails] = Seq(
/**
* Production clients for timelinemixer.
*/
new ClientDetails(
"timelinemixer.recap.prod",
RateLimitingTrustedPermission(AllowTimelineMixerRecapProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinemixer.recycled.prod",
RateLimitingTrustedPermission(AllowTimelineMixerRecycledProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinemixer.hydrate.prod",
RateLimitingTrustedPermission(AllowTimelineMixerHydrateProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinemixer.hydrate_recos.prod",
RateLimitingTrustedPermission(AllowTimelineMixerHydrateRecosProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinemixer.seed_author_ids.prod",
RateLimitingTrustedPermission(AllowTimelineMixerSeedAuthorsProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinemixer.simcluster.prod",
RateLimitingTrustedPermission(AllowTimelineMixerSimclusterProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinemixer.entity_tweets.prod",
RateLimitingTrustedPermission(AllowTimelineMixerEntityTweetsProd),
protectedWriteAccess = TrustedPermission
),
/**
* This client is whitelisted for timelinemixer only as it used by
* List injection service which will not be migrated to timelinescorer.
*/
new ClientDetails(
"timelinemixer.list.prod",
RateLimitingTrustedPermission(AllowTimelineMixerListProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinemixer.list_tweet.prod",
RateLimitingTrustedPermission(AllowTimelineMixerListTweetProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinemixer.community.prod",
RateLimitingTrustedPermission(AllowTimelineMixerCommunityProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinemixer.community_tweet.prod",
RateLimitingTrustedPermission(AllowTimelineMixerCommunityTweetProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinemixer.uteg_liked_by_tweets.prod",
RateLimitingTrustedPermission(AllowTimelineMixerUtegLikedByTweetsProd),
protectedWriteAccess = TrustedPermission
),
/**
* Production clients for timelinescorer. Most of these clients have their
* equivalents under the timelinemixer scope (with exception of list injection
* client).
*/
new ClientDetails(
"timelinescorer.recap.prod",
RateLimitingTrustedPermission(AllowTimelineMixerRecapProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinescorer.recycled.prod",
RateLimitingTrustedPermission(AllowTimelineMixerRecycledProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinescorer.hydrate.prod",
RateLimitingTrustedPermission(AllowTimelineMixerHydrateProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinescorer.hydrate_recos.prod",
RateLimitingTrustedPermission(AllowTimelineMixerHydrateRecosProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinescorer.seed_author_ids.prod",
RateLimitingTrustedPermission(AllowTimelineMixerSeedAuthorsProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinescorer.simcluster.prod",
RateLimitingTrustedPermission(AllowTimelineMixerSimclusterProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinescorer.entity_tweets.prod",
RateLimitingTrustedPermission(AllowTimelineMixerEntityTweetsProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinescorer.list_tweet.prod",
RateLimitingTrustedPermission(AllowTimelineMixerListTweetProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinescorer.uteg_liked_by_tweets.prod",
RateLimitingTrustedPermission(AllowTimelineMixerUtegLikedByTweetsProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelineservice.prod",
RateLimitingTrustedPermission(AllowTimelineServiceProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinescorer.hydrate_tweet_scoring.prod",
RateLimitingTrustedPermission(AllowTimelineScorerHydrateTweetScoringProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinescorer.community_tweet.prod",
RateLimitingTrustedPermission(AllowTimelineMixerCommunityTweetProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinescorer.recommended_trend_tweet.prod",
RateLimitingTrustedPermission(AllowTimelineScorerRecommendedTrendTweetProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinescorer.rec_topic_tweets.prod",
RateLimitingTrustedPermission(AllowTimelineScorerRecTopicTweetsProd),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinescorer.popular_topic_tweets.prod",
RateLimitingTrustedPermission(AllowTimelineScorerPopularTopicTweetsProd),
protectedWriteAccess = TrustedPermission
),
/**
* TimelineRanker utilities. Traffic proxy, warmups, and console.
*/
new ClientDetails(
"timelineranker.proxy",
RateLimitingTrustedPermission(AllowTimelineRankerProxy),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
TimelineRankerConstants.WarmupClientName,
RateLimitingTrustedPermission(AllowTimelineRankerWarmup),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
TimelineRankerConstants.ForwardedClientName,
RateLimitingTrustedPermission(AllowTimelineRankerWarmup),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelineranker.console",
RateLimitingUntrustedPermission(RateLimitOverrideUnknown, 1d),
protectedWriteAccess = TrustedPermission
),
/**
* Staging clients.
*/
new ClientDetails(
"timelinemixer.recap.staging",
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinemixer.recycled.staging",
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinemixer.hydrate.staging",
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinemixer.hydrate_recos.staging",
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinemixer.seed_author_ids.staging",
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinemixer.simcluster.staging",
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinemixer.entity_tweets.staging",
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinemixer.list.staging",
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinemixer.list_tweet.staging",
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinemixer.community.staging",
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinemixer.community_tweet.staging",
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinescorer.community_tweet.staging",
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinescorer.recommended_trend_tweet.staging",
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinemixer.uteg_liked_by_tweets.staging",
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinemixer.entity_tweets.staging",
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinescorer.hydrate_tweet_scoring.staging",
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinescorer.rec_topic_tweets.staging",
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelinescorer.popular_topic_tweets.staging",
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
protectedWriteAccess = TrustedPermission
),
new ClientDetails(
"timelineservice.staging",
RateLimitingTrustedPermission(AllowTimelineServiceStaging),
protectedWriteAccess = TrustedPermission
)
)
}

86
timelineranker/server/src/main/scala/com/twitter/timelineranker/config/ClientWrapperFactories.scala Normal file
View File

@ -0,0 +1,86 @@
package com.twitter.timelineranker.config
import com.twitter.servo.util.Gate
import com.twitter.timelineranker.clients.ScopedCortexTweetQueryServiceClientFactory
import com.twitter.timelines.clients.gizmoduck.ScopedGizmoduckClientFactory
import com.twitter.timelines.clients.manhattan.ScopedUserMetadataClientFactory
import com.twitter.timelines.clients.socialgraph.ScopedSocialGraphClientFactory
import com.twitter.timelines.clients.strato.realgraph.ScopedRealGraphClientFactory
import com.twitter.timelines.clients.tweetypie.AdditionalFieldConfig
import com.twitter.timelines.clients.tweetypie.ScopedTweetyPieClientFactory
import com.twitter.timelines.visibility.VisibilityEnforcerFactory
import com.twitter.timelines.visibility.VisibilityProfileHydratorFactory
import com.twitter.tweetypie.thriftscala.{Tweet => TTweet}
class ClientWrapperFactories(config: RuntimeConfiguration) {
private[this] val statsReceiver = config.statsReceiver
val cortexTweetQueryServiceClientFactory: ScopedCortexTweetQueryServiceClientFactory =
new ScopedCortexTweetQueryServiceClientFactory(
config.underlyingClients.cortexTweetQueryServiceClient,
statsReceiver = statsReceiver
)
val gizmoduckClientFactory: ScopedGizmoduckClientFactory = new ScopedGizmoduckClientFactory(
config.underlyingClients.gizmoduckClient,
statsReceiver = statsReceiver
)
val socialGraphClientFactory: ScopedSocialGraphClientFactory = new ScopedSocialGraphClientFactory(
config.underlyingClients.sgsClient,
statsReceiver
)
val visibilityEnforcerFactory: VisibilityEnforcerFactory = new VisibilityEnforcerFactory(
gizmoduckClientFactory,
socialGraphClientFactory,
statsReceiver
)
val tweetyPieAdditionalFieldsToDisable: Seq[Short] = Seq(
TTweet.MediaTagsField.id,
TTweet.SchedulingInfoField.id,
TTweet.EscherbirdEntityAnnotationsField.id,
TTweet.CardReferenceField.id,
TTweet.SelfPermalinkField.id,
TTweet.ExtendedTweetMetadataField.id,
TTweet.CommunitiesField.id,
TTweet.VisibleTextRangeField.id
)
val tweetyPieHighQoSClientFactory: ScopedTweetyPieClientFactory =
new ScopedTweetyPieClientFactory(
tweetyPieClient = config.underlyingClients.tweetyPieHighQoSClient,
additionalFieldConfig = AdditionalFieldConfig(
fieldDisablingGates = tweetyPieAdditionalFieldsToDisable.map(_ -> Gate.False).toMap
),
includePartialResults = Gate.False,
statsReceiver = statsReceiver
)
val tweetyPieLowQoSClientFactory: ScopedTweetyPieClientFactory = new ScopedTweetyPieClientFactory(
tweetyPieClient = config.underlyingClients.tweetyPieLowQoSClient,
additionalFieldConfig = AdditionalFieldConfig(
fieldDisablingGates = tweetyPieAdditionalFieldsToDisable.map(_ -> Gate.False).toMap
),
includePartialResults = Gate.False,
statsReceiver = statsReceiver
)
val userMetadataClientFactory: ScopedUserMetadataClientFactory =
new ScopedUserMetadataClientFactory(
config.underlyingClients.manhattanStarbuckClient,
TimelineRankerConstants.ManhattanStarbuckAppId,
statsReceiver
)
val visibilityProfileHydratorFactory: VisibilityProfileHydratorFactory =
new VisibilityProfileHydratorFactory(
gizmoduckClientFactory,
socialGraphClientFactory,
statsReceiver
)
val realGraphClientFactory =
new ScopedRealGraphClientFactory(config.underlyingClients.stratoClient, statsReceiver)
}

11
timelineranker/server/src/main/scala/com/twitter/timelineranker/config/ClientWrappers.scala Normal file
View File

@ -0,0 +1,11 @@
package com.twitter.timelineranker.config
import com.twitter.storehaus.Store
import com.twitter.timelineranker.recap.model.ContentFeatures
import com.twitter.timelines.model.TweetId
class ClientWrappers(config: RuntimeConfiguration) {
private[this] val backendClientConfiguration = config.underlyingClients
val contentFeaturesCache: Store[TweetId, ContentFeatures] =
backendClientConfiguration.contentFeaturesCache
}

158
timelineranker/server/src/main/scala/com/twitter/timelineranker/config/DefaultUnderlyingClientConfiguration.scala Normal file
View File

@ -0,0 +1,158 @@
package com.twitter.timelineranker.config
import com.twitter.conversions.DurationOps._
import com.twitter.conversions.PercentOps._
import com.twitter.cortex_tweet_annotate.thriftscala.CortexTweetQueryService
import com.twitter.finagle.ssl.OpportunisticTls
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.finagle.thrift.ClientId
import com.twitter.finagle.util.DefaultTimer
import com.twitter.gizmoduck.thriftscala.{UserService => Gizmoduck}
import com.twitter.manhattan.v1.thriftscala.{ManhattanCoordinator => ManhattanV1}
import com.twitter.merlin.thriftscala.UserRolesService
import com.twitter.recos.user_tweet_entity_graph.thriftscala.UserTweetEntityGraph
import com.twitter.socialgraph.thriftscala.SocialGraphService
import com.twitter.storehaus.Store
import com.twitter.strato.client.Strato
import com.twitter.strato.client.{Client => StratoClient}
import com.twitter.timelineranker.clients.content_features_cache.ContentFeaturesMemcacheBuilder
import com.twitter.timelineranker.recap.model.ContentFeatures
import com.twitter.timelineranker.thriftscala.TimelineRanker
import com.twitter.timelines.clients.memcache_common.StorehausMemcacheConfig
import com.twitter.timelines.model.TweetId
import com.twitter.timelineservice.thriftscala.TimelineService
import com.twitter.tweetypie.thriftscala.{TweetService => TweetyPie}
import com.twitter.util.Timer
import org.apache.thrift.protocol.TCompactProtocol
class DefaultUnderlyingClientConfiguration(flags: TimelineRankerFlags, statsReceiver: StatsReceiver)
extends UnderlyingClientConfiguration(flags, statsReceiver) { top =>
val timer: Timer = DefaultTimer
val twCachePrefix = "/srv#/prod/local/cache"
override val cortexTweetQueryServiceClient: CortexTweetQueryService.MethodPerEndpoint = {
methodPerEndpointClient[
CortexTweetQueryService.ServicePerEndpoint,
CortexTweetQueryService.MethodPerEndpoint](
thriftMuxClientBuilder("cortex-tweet-query", requireMutualTls = true)
.dest("/s/cortex-tweet-annotate/cortex-tweet-query")
.requestTimeout(200.milliseconds)
.timeout(500.milliseconds)
)
}
override val gizmoduckClient: Gizmoduck.MethodPerEndpoint = {
methodPerEndpointClient[Gizmoduck.ServicePerEndpoint, Gizmoduck.MethodPerEndpoint](
thriftMuxClientBuilder("gizmoduck", requireMutualTls = true)
.dest("/s/gizmoduck/gizmoduck")
.requestTimeout(400.milliseconds)
.timeout(900.milliseconds)
)
}
override lazy val manhattanStarbuckClient: ManhattanV1.MethodPerEndpoint = {
methodPerEndpointClient[ManhattanV1.ServicePerEndpoint, ManhattanV1.MethodPerEndpoint](
thriftMuxClientBuilder("manhattan.starbuck", requireMutualTls = true)
.dest("/s/manhattan/starbuck.native-thrift")
.requestTimeout(600.millis)
)
}
override val sgsClient: SocialGraphService.MethodPerEndpoint = {
methodPerEndpointClient[
SocialGraphService.ServicePerEndpoint,
SocialGraphService.MethodPerEndpoint](
thriftMuxClientBuilder("socialgraph", requireMutualTls = true)
.dest("/s/socialgraph/socialgraph")
.requestTimeout(350.milliseconds)
.timeout(700.milliseconds)
)
}
override lazy val timelineRankerForwardingClient: TimelineRanker.FinagledClient =
new TimelineRanker.FinagledClient(
thriftMuxClientBuilder(
TimelineRankerConstants.ForwardedClientName,
ClientId(TimelineRankerConstants.ForwardedClientName),
protocolFactoryOption = Some(new TCompactProtocol.Factory()),
requireMutualTls = true
).dest("/s/timelineranker/timelineranker:compactthrift").build(),
protocolFactory = new TCompactProtocol.Factory()
)
override val timelineServiceClient: TimelineService.MethodPerEndpoint = {
methodPerEndpointClient[TimelineService.ServicePerEndpoint, TimelineService.MethodPerEndpoint](
thriftMuxClientBuilder("timelineservice", requireMutualTls = true)
.dest("/s/timelineservice/timelineservice")
.requestTimeout(600.milliseconds)
.timeout(800.milliseconds)
)
}
override val tweetyPieHighQoSClient: TweetyPie.MethodPerEndpoint = {
methodPerEndpointClient[TweetyPie.ServicePerEndpoint, TweetyPie.MethodPerEndpoint](
thriftMuxClientBuilder("tweetypieHighQoS", requireMutualTls = true)
.dest("/s/tweetypie/tweetypie")
.requestTimeout(450.milliseconds)
.timeout(800.milliseconds),
maxExtraLoadPercent = Some(1.percent)
)
}
/**
* Provide less costly TweetPie client with the trade-off of reduced quality. Intended for use cases
* which are not essential for successful completion of timeline requests. Currently this client differs
* from the highQoS endpoint by having retries count set to 1 instead of 2.
*/
override val tweetyPieLowQoSClient: TweetyPie.MethodPerEndpoint = {
methodPerEndpointClient[TweetyPie.ServicePerEndpoint, TweetyPie.MethodPerEndpoint](
thriftMuxClientBuilder("tweetypieLowQoS", requireMutualTls = true)
.dest("/s/tweetypie/tweetypie")
.retryPolicy(mkRetryPolicy(1)) // override default value
.requestTimeout(450.milliseconds)
.timeout(800.milliseconds),
maxExtraLoadPercent = Some(1.percent)
)
}
override val userRolesServiceClient: UserRolesService.MethodPerEndpoint = {
methodPerEndpointClient[
UserRolesService.ServicePerEndpoint,
UserRolesService.MethodPerEndpoint](
thriftMuxClientBuilder("merlin", requireMutualTls = true)
.dest("/s/merlin/merlin")
.requestTimeout(1.second)
)
}
lazy val contentFeaturesCache: Store[TweetId, ContentFeatures] =
new ContentFeaturesMemcacheBuilder(
config = new StorehausMemcacheConfig(
destName = s"$twCachePrefix/timelines_content_features:twemcaches",
keyPrefix = "",
requestTimeout = 50.milliseconds,
numTries = 1,
globalTimeout = 60.milliseconds,
tcpConnectTimeout = 50.milliseconds,
connectionAcquisitionTimeout = 25.milliseconds,
numPendingRequests = 100,
isReadOnly = false,
serviceIdentifier = serviceIdentifier
),
ttl = 48.hours,
statsReceiver
).build
override val userTweetEntityGraphClient: UserTweetEntityGraph.FinagledClient =
new UserTweetEntityGraph.FinagledClient(
thriftMuxClientBuilder("user_tweet_entity_graph", requireMutualTls = true)
.dest("/s/cassowary/user_tweet_entity_graph")
.retryPolicy(mkRetryPolicy(2))
.requestTimeout(300.milliseconds)
.build()
)
override val stratoClient: StratoClient =
Strato.client.withMutualTls(serviceIdentifier, OpportunisticTls.Required).build()
}

13
timelineranker/server/src/main/scala/com/twitter/timelineranker/config/RequestScopes.scala Normal file
View File

@ -0,0 +1,13 @@
package com.twitter.timelineranker.config
import com.twitter.timelines.util.stats.RequestScope
object RequestScopes {
val HomeTimelineMaterialization: RequestScope = RequestScope("homeMaterialization")
val InNetworkTweetSource: RequestScope = RequestScope("inNetworkTweet")
val RecapHydrationSource: RequestScope = RequestScope("recapHydration")
val RecapAuthorSource: RequestScope = RequestScope("recapAuthor")
val ReverseChronHomeTimelineSource: RequestScope = RequestScope("reverseChronHomeTimelineSource")
val EntityTweetsSource: RequestScope = RequestScope("entityTweets")
val UtegLikedByTweetsSource: RequestScope = RequestScope("utegLikedByTweets")
}

133
timelineranker/server/src/main/scala/com/twitter/timelineranker/config/RuntimeConfiguration.scala Normal file
View File

@ -0,0 +1,133 @@
package com.twitter.timelineranker.config
import com.twitter.abdecider.ABDeciderFactory
import com.twitter.abdecider.LoggingABDecider
import com.twitter.decider.Decider
import com.twitter.featureswitches.Value
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.servo.decider.DeciderGateBuilder
import com.twitter.servo.util.Effect
import com.twitter.timelineranker.decider.DeciderKey
import com.twitter.timelines.authorization.TimelinesClientRequestAuthorizer
import com.twitter.timelines.config._
import com.twitter.timelines.config.configapi._
import com.twitter.timelines.features._
import com.twitter.timelines.util.ImpressionCountingABDecider
import com.twitter.timelines.util.logging.Scribe
import com.twitter.util.Await
import com.twitter.servo.util.Gate
import com.twitter.timelines.model.UserId
trait ClientProvider {
def clientWrappers: ClientWrappers
def underlyingClients: UnderlyingClientConfiguration
}
trait UtilityProvider {
def abdecider: LoggingABDecider
def clientRequestAuthorizer: TimelinesClientRequestAuthorizer
def configStore: ConfigStore
def decider: Decider
def deciderGateBuilder: DeciderGateBuilder
def employeeGate: UserRolesGate.EmployeeGate
def configApiConfiguration: ConfigApiConfiguration
def statsReceiver: StatsReceiver
def whitelist: UserList
}
trait RuntimeConfiguration extends ClientProvider with UtilityProvider with ConfigUtils {
def isProd: Boolean
def maxConcurrency: Int
def clientEventScribe: Effect[String]
def clientWrapperFactories: ClientWrapperFactories
}
class RuntimeConfigurationImpl(
flags: TimelineRankerFlags,
configStoreFactory: DynamicConfigStoreFactory,
val decider: Decider,
val forcedFeatureValues: Map[String, Value] = Map.empty[String, Value],
val statsReceiver: StatsReceiver)
extends RuntimeConfiguration {
// Creates and initialize config store as early as possible so other parts could have a dependency on it for settings.
override val configStore: DynamicConfigStore =
configStoreFactory.createDcEnvAwareFileBasedConfigStore(
relativeConfigFilePath = "timelines/timelineranker/service_settings.yml",
dc = flags.getDatacenter,
env = flags.getEnv,
configBusConfig = ConfigBusProdConfig,
onUpdate = ConfigStore.NullOnUpdateCallback,
statsReceiver = statsReceiver
)
Await.result(configStore.init)
val environment: Env.Value = flags.getEnv
override val isProd: Boolean = isProdEnv(environment)
val datacenter: Datacenter.Value = flags.getDatacenter
val abDeciderPath = "/usr/local/config/abdecider/abdecider.yml"
override val maxConcurrency: Int = flags.maxConcurrency()
val deciderGateBuilder: DeciderGateBuilder = new DeciderGateBuilder(decider)
val clientRequestAuthorizer: TimelinesClientRequestAuthorizer =
new TimelinesClientRequestAuthorizer(
deciderGateBuilder = deciderGateBuilder,
clientDetails = ClientAccessPermissions.All,
unknownClientDetails = ClientAccessPermissions.unknown,
clientAuthorizationGate =
deciderGateBuilder.linearGate(DeciderKey.ClientRequestAuthorization),
clientWriteWhitelistGate = deciderGateBuilder.linearGate(DeciderKey.ClientWriteWhitelist),
globalCapacityQPS = flags.requestRateLimit(),
statsReceiver = statsReceiver
)
override val clientEventScribe = Scribe.clientEvent(isProd, statsReceiver)
val abdecider: LoggingABDecider = new ImpressionCountingABDecider(
abdecider = ABDeciderFactory.withScribeEffect(
abDeciderYmlPath = abDeciderPath,
scribeEffect = clientEventScribe,
decider = None,
environment = Some("production"),
).buildWithLogging(),
statsReceiver = statsReceiver
)
val underlyingClients: UnderlyingClientConfiguration = buildUnderlyingClientConfiguration
val clientWrappers: ClientWrappers = new ClientWrappers(this)
override val clientWrapperFactories: ClientWrapperFactories = new ClientWrapperFactories(this)
private[this] val userRolesCacheFactory = new UserRolesCacheFactory(
userRolesService = underlyingClients.userRolesServiceClient,
statsReceiver = statsReceiver
)
override val whitelist: Whitelist = Whitelist(
configStoreFactory = configStoreFactory,
userRolesCacheFactory = userRolesCacheFactory,
statsReceiver = statsReceiver
)
override val employeeGate: Gate[UserId] = UserRolesGate(
userRolesCacheFactory.create(UserRoles.EmployeesRoleName)
)
private[this] val featureRecipientFactory =
new UserRolesCachingFeatureRecipientFactory(userRolesCacheFactory, statsReceiver)
override val configApiConfiguration: FeatureSwitchesV2ConfigApiConfiguration =
FeatureSwitchesV2ConfigApiConfiguration(
datacenter = flags.getDatacenter,
serviceName = ServiceName.TimelineRanker,
abdecider = abdecider,
featureRecipientFactory = featureRecipientFactory,
forcedValues = forcedFeatureValues,
statsReceiver = statsReceiver
)
private[this] def buildUnderlyingClientConfiguration: UnderlyingClientConfiguration = {
environment match {
case Env.prod => new DefaultUnderlyingClientConfiguration(flags, statsReceiver)
case _ => new StagingUnderlyingClientConfiguration(flags, statsReceiver)
}
}
}

6
timelineranker/server/src/main/scala/com/twitter/timelineranker/config/StagingUnderlyingConfiguration.scala Normal file
View File

@ -0,0 +1,6 @@
package com.twitter.timelineranker.config
import com.twitter.finagle.stats.StatsReceiver
class StagingUnderlyingClientConfiguration(flags: TimelineRankerFlags, statsReceiver: StatsReceiver)
extends DefaultUnderlyingClientConfiguration(flags, statsReceiver)

8
timelineranker/server/src/main/scala/com/twitter/timelineranker/config/TimelineRankerConstants.scala Normal file
View File

@ -0,0 +1,8 @@
package com.twitter.timelineranker.config
object TimelineRankerConstants {
val ClientPrefix = "timelineranker."
val ManhattanStarbuckAppId = "timelineranker"
val WarmupClientName = "timelineranker.warmup"
val ForwardedClientName = "timelineranker.forwarded"
}

72
timelineranker/server/src/main/scala/com/twitter/timelineranker/config/TimelineRankerFlags.scala Normal file
View File

@ -0,0 +1,72 @@
package com.twitter.timelineranker.config
import com.twitter.app.Flags
import com.twitter.finagle.mtls.authentication.EmptyServiceIdentifier
import com.twitter.finagle.mtls.authentication.ServiceIdentifier
import com.twitter.timelines.config.CommonFlags
import com.twitter.timelines.config.ConfigUtils
import com.twitter.timelines.config.Datacenter
import com.twitter.timelines.config.Env
import com.twitter.timelines.config.ProvidesServiceIdentifier
import java.net.InetSocketAddress
import com.twitter.app.Flag
class TimelineRankerFlags(flag: Flags)
extends CommonFlags(flag)
with ConfigUtils
with ProvidesServiceIdentifier {
val dc: Flag[String] = flag(
"dc",
"smf1",
"Name of data center in which this instance will execute"
)
val environment: Flag[String] = flag(
"environment",
"devel",
"The mesos environment in which this instance will be running"
)
val maxConcurrency: Flag[Int] = flag(
"maxConcurrency",
200,
"Maximum concurrent requests"
)
val servicePort: Flag[InetSocketAddress] = flag(
"service.port",
new InetSocketAddress(8287),
"Port number that this thrift service will listen on"
)
val serviceCompactPort: Flag[InetSocketAddress] = flag(
"service.compact.port",
new InetSocketAddress(8288),
"Port number that the TCompactProtocol-based thrift service will listen on"
)
val serviceIdentifier: Flag[ServiceIdentifier] = flag[ServiceIdentifier](
"service.identifier",
EmptyServiceIdentifier,
"service identifier for this service for use with mutual TLS, " +
"format is expected to be -service.identifier=\"role:service:environment:zone\""
)
val opportunisticTlsLevel = flag[String](
"opportunistic.tls.level",
"desired",
"The server's OpportunisticTls level."
)
val requestRateLimit: Flag[Double] = flag[Double](
"requestRateLimit",
1000.0,
"Request rate limit to be used by the client request authorizer"
)
val enableThriftmuxCompression = flag(
"enableThriftmuxServerCompression",
true,
"build server with thriftmux compression enabled"
)
def getDatacenter: Datacenter.Value = getDC(dc())
def getEnv: Env.Value = getEnv(environment())
override def getServiceIdentifier: ServiceIdentifier = serviceIdentifier()
}

107
timelineranker/server/src/main/scala/com/twitter/timelineranker/config/UnderlyingClientConfiguration.scala Normal file
View File

@ -0,0 +1,107 @@
package com.twitter.timelineranker.config
import com.twitter.cortex_tweet_annotate.thriftscala.CortexTweetQueryService
import com.twitter.finagle.Service
import com.twitter.finagle.mtls.authentication.ServiceIdentifier
import com.twitter.finagle.service.RetryPolicy
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.finagle.thrift.ClientId
import com.twitter.finagle.thrift.ThriftClientRequest
import com.twitter.gizmoduck.thriftscala.{UserService => Gizmoduck}
import com.twitter.manhattan.v1.thriftscala.{ManhattanCoordinator => ManhattanV1}
import com.twitter.merlin.thriftscala.UserRolesService
import com.twitter.recos.user_tweet_entity_graph.thriftscala.UserTweetEntityGraph
import com.twitter.search.earlybird.thriftscala.EarlybirdService
import com.twitter.socialgraph.thriftscala.SocialGraphService
import com.twitter.storehaus.Store
import com.twitter.strato.client.{Client => StratoClient}
import com.twitter.timelineranker.recap.model.ContentFeatures
import com.twitter.timelineranker.thriftscala.TimelineRanker
import com.twitter.timelines.config.ConfigUtils
import com.twitter.timelines.config.TimelinesUnderlyingClientConfiguration
import com.twitter.timelines.model.TweetId
import com.twitter.timelineservice.thriftscala.TimelineService
import com.twitter.tweetypie.thriftscala.{TweetService => TweetyPie}
import com.twitter.util.Duration
import com.twitter.util.Try
import org.apache.thrift.protocol.TCompactProtocol
abstract class UnderlyingClientConfiguration(
flags: TimelineRankerFlags,
val statsReceiver: StatsReceiver)
extends TimelinesUnderlyingClientConfiguration
with ConfigUtils {
lazy val thriftClientId: ClientId = timelineRankerClientId()
override lazy val serviceIdentifier: ServiceIdentifier = flags.getServiceIdentifier
def timelineRankerClientId(scope: Option[String] = None): ClientId = {
clientIdWithScopeOpt("timelineranker", flags.getEnv, scope)
}
def createEarlybirdClient(
scope: String,
requestTimeout: Duration,
timeout: Duration,
retryPolicy: RetryPolicy[Try[Nothing]]
): EarlybirdService.MethodPerEndpoint = {
val scopedName = s"earlybird/$scope"
methodPerEndpointClient[
EarlybirdService.ServicePerEndpoint,
EarlybirdService.MethodPerEndpoint](
thriftMuxClientBuilder(
scopedName,
protocolFactoryOption = Some(new TCompactProtocol.Factory),
requireMutualTls = true)
.dest("/s/earlybird-root-superroot/root-superroot")
.timeout(timeout)
.requestTimeout(requestTimeout)
.retryPolicy(retryPolicy)
)
}
def createEarlybirdRealtimeCgClient(
scope: String,
requestTimeout: Duration,
timeout: Duration,
retryPolicy: RetryPolicy[Try[Nothing]]
): EarlybirdService.MethodPerEndpoint = {
val scopedName = s"earlybird/$scope"
methodPerEndpointClient[
EarlybirdService.ServicePerEndpoint,
EarlybirdService.MethodPerEndpoint](
thriftMuxClientBuilder(
scopedName,
protocolFactoryOption = Some(new TCompactProtocol.Factory),
requireMutualTls = true)
.dest("/s/earlybird-rootrealtimecg/root-realtime_cg")
.timeout(timeout)
.requestTimeout(requestTimeout)
.retryPolicy(retryPolicy)
)
}
def cortexTweetQueryServiceClient: CortexTweetQueryService.MethodPerEndpoint
def gizmoduckClient: Gizmoduck.MethodPerEndpoint
def manhattanStarbuckClient: ManhattanV1.MethodPerEndpoint
def sgsClient: SocialGraphService.MethodPerEndpoint
def timelineRankerForwardingClient: TimelineRanker.FinagledClient
def timelineServiceClient: TimelineService.MethodPerEndpoint
def tweetyPieHighQoSClient: TweetyPie.MethodPerEndpoint
def tweetyPieLowQoSClient: TweetyPie.MethodPerEndpoint
def userRolesServiceClient: UserRolesService.MethodPerEndpoint
def contentFeaturesCache: Store[TweetId, ContentFeatures]
def userTweetEntityGraphClient: UserTweetEntityGraph.FinagledClient
def stratoClient: StratoClient
def darkTrafficProxy: Option[Service[ThriftClientRequest, Array[Byte]]] = {
if (flags.darkTrafficDestination.trim.nonEmpty) {
Some(darkTrafficClient(flags.darkTrafficDestination))
} else {
None
}
}
}

13
timelineranker/server/src/main/scala/com/twitter/timelineranker/contentfeatures/BUILD Normal file
View File

@ -0,0 +1,13 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
platform = "java8",
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"timelineranker/common/src/main/scala/com/twitter/timelineranker/model",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/core",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model",
"timelines/src/main/scala/com/twitter/timelines/model/types",
],
)

10
timelineranker/server/src/main/scala/com/twitter/timelineranker/contentfeatures/package.scala Normal file
View File

@ -0,0 +1,10 @@
package com.twitter.timelineranker
import com.twitter.timelineranker.core.FutureDependencyTransformer
import com.twitter.timelineranker.recap.model.ContentFeatures
import com.twitter.timelines.model.TweetId
package object contentfeatures {
type ContentFeaturesProvider =
FutureDependencyTransformer[Seq[TweetId], Map[TweetId, ContentFeatures]]
}

24
timelineranker/server/src/main/scala/com/twitter/timelineranker/core/BUILD Normal file
View File

@ -0,0 +1,24 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
platform = "java8",
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
"src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
"src/thrift/com/twitter/search:earlybird-scala",
"src/thrift/com/twitter/search/common:constants-scala",
"src/thrift/com/twitter/search/common:features-scala",
"timelineranker/common:model",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model",
"timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck",
"timelines/src/main/scala/com/twitter/timelines/model/tweet",
"timelines/src/main/scala/com/twitter/timelines/util",
"timelines/src/main/scala/com/twitter/timelines/visibility/model",
"util/util-core:util-core-util",
],
exports = [
"src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
],
)

24
timelineranker/server/src/main/scala/com/twitter/timelineranker/core/CandidateEnvelope.scala Normal file
View File

@ -0,0 +1,24 @@
package com.twitter.timelineranker.core
import com.twitter.recos.user_tweet_entity_graph.thriftscala.TweetRecommendation
import com.twitter.search.earlybird.thriftscala.ThriftSearchResult
import com.twitter.timelineranker.model.RecapQuery
import com.twitter.timelines.model.TweetId
object CandidateEnvelope {
val EmptySearchResults: Seq[ThriftSearchResult] = Seq.empty[ThriftSearchResult]
val EmptyHydratedTweets: HydratedTweets = HydratedTweets(Seq.empty, Seq.empty)
val EmptyUtegResults: Map[TweetId, TweetRecommendation] = Map.empty[TweetId, TweetRecommendation]
}
case class CandidateEnvelope(
query: RecapQuery,
searchResults: Seq[ThriftSearchResult] = CandidateEnvelope.EmptySearchResults,
utegResults: Map[TweetId, TweetRecommendation] = CandidateEnvelope.EmptyUtegResults,
hydratedTweets: HydratedTweets = CandidateEnvelope.EmptyHydratedTweets,
followGraphData: FollowGraphDataFuture = FollowGraphDataFuture.EmptyFollowGraphDataFuture,
// The source tweets are
// - the retweeted tweet, for retweets
// - the inReplyTo tweet, for extended replies
sourceSearchResults: Seq[ThriftSearchResult] = CandidateEnvelope.EmptySearchResults,
sourceHydratedTweets: HydratedTweets = CandidateEnvelope.EmptyHydratedTweets)

34
timelineranker/server/src/main/scala/com/twitter/timelineranker/core/FollowGraphData.scala Normal file
View File

@ -0,0 +1,34 @@
package com.twitter.timelineranker.core
import com.twitter.timelines.model.UserId
/**
* Follow graph details of a given user. Includes users followed, but also followed users in various
* states of mute.
*
* @param userId ID of a given user.
* @param followedUserIds IDs of users who the given user follows.
* @param mutuallyFollowingUserIds A subset of followedUserIds where followed users follow back the given user.
* @param mutedUserIds A subset of followedUserIds that the given user has muted.
* @param retweetsMutedUserIds A subset of followedUserIds whose retweets are muted by the given user.
*/
case class FollowGraphData(
userId: UserId,
followedUserIds: Seq[UserId],
mutuallyFollowingUserIds: Set[UserId],
mutedUserIds: Set[UserId],
retweetsMutedUserIds: Set[UserId]) {
val filteredFollowedUserIds: Seq[UserId] = followedUserIds.filterNot(mutedUserIds)
val allUserIds: Seq[UserId] = filteredFollowedUserIds :+ userId
val inNetworkUserIds: Seq[UserId] = followedUserIds :+ userId
}
object FollowGraphData {
val Empty: FollowGraphData = FollowGraphData(
0L,
Seq.empty[UserId],
Set.empty[UserId],
Set.empty[UserId],
Set.empty[UserId]
)
}

53
timelineranker/server/src/main/scala/com/twitter/timelineranker/core/FollowGraphDataFuture.scala Normal file
View File

@ -0,0 +1,53 @@
package com.twitter.timelineranker.core
import com.twitter.timelines.model.UserId
import com.twitter.util.Future
/**
* Similar to FollowGraphData but makes available its fields as separate futures.
*/
case class FollowGraphDataFuture(
userId: UserId,
followedUserIdsFuture: Future[Seq[UserId]],
mutuallyFollowingUserIdsFuture: Future[Set[UserId]],
mutedUserIdsFuture: Future[Set[UserId]],
retweetsMutedUserIdsFuture: Future[Set[UserId]]) {
def inNetworkUserIdsFuture: Future[Seq[UserId]] = followedUserIdsFuture.map(_ :+ userId)
def get(): Future[FollowGraphData] = {
Future
.join(
followedUserIdsFuture,
mutuallyFollowingUserIdsFuture,
mutedUserIdsFuture,
retweetsMutedUserIdsFuture
)
.map {
case (followedUserIds, mutuallyFollowingUserIds, mutedUserIds, retweetsMutedUserIds) =>
FollowGraphData(
userId,
followedUserIds,
mutuallyFollowingUserIds,
mutedUserIds,
retweetsMutedUserIds
)
}
}
}
object FollowGraphDataFuture {
private def mkEmptyFuture(field: String) = {
Future.exception(
new IllegalStateException(s"FollowGraphDataFuture field $field accessed without being set")
)
}
val EmptyFollowGraphDataFuture: FollowGraphDataFuture = FollowGraphDataFuture(
userId = 0L,
followedUserIdsFuture = mkEmptyFuture("followedUserIdsFuture"),
mutuallyFollowingUserIdsFuture = mkEmptyFuture("mutuallyFollowingUserIdsFuture"),
mutedUserIdsFuture = mkEmptyFuture("mutedUserIdsFuture"),
retweetsMutedUserIdsFuture = mkEmptyFuture("retweetsMutedUserIdsFuture")
)
}

18
timelineranker/server/src/main/scala/com/twitter/timelineranker/core/HydratedCandidatesAndFeaturesEnvelope.scala Normal file
View File

@ -0,0 +1,18 @@
package com.twitter.timelineranker.core
import com.twitter.search.common.constants.thriftscala.ThriftLanguage
import com.twitter.search.common.features.thriftscala.ThriftTweetFeatures
import com.twitter.timelineranker.recap.model.ContentFeatures
import com.twitter.timelines.clients.gizmoduck.UserProfileInfo
import com.twitter.timelines.model.TweetId
import com.twitter.timelines.util.FutureUtils
import com.twitter.util.Future
case class HydratedCandidatesAndFeaturesEnvelope(
candidateEnvelope: CandidateEnvelope,
userLanguages: Seq[ThriftLanguage],
userProfileInfo: UserProfileInfo,
features: Map[TweetId, ThriftTweetFeatures] = Map.empty,
contentFeaturesFuture: Future[Map[TweetId, ContentFeatures]] = FutureUtils.EmptyMap,
tweetSourceTweetMap: Map[TweetId, TweetId] = Map.empty,
inReplyToTweetIds: Set[TweetId] = Set.empty)

7
timelineranker/server/src/main/scala/com/twitter/timelineranker/core/HydratedTweets.scala Normal file
View File

@ -0,0 +1,7 @@
package com.twitter.timelineranker.core
import com.twitter.timelines.model.tweet.HydratedTweet
case class HydratedTweets(
outerTweets: Seq[HydratedTweet],
innerTweets: Seq[HydratedTweet] = Seq.empty)

13
timelineranker/server/src/main/scala/com/twitter/timelineranker/core/package.scala Normal file
View File

@ -0,0 +1,13 @@
package com.twitter.timelineranker
import com.twitter.timelineranker.model.RecapQuery
import com.twitter.timelines.configapi
package object core {
type FutureDependencyTransformer[-U, +V] = configapi.FutureDependencyTransformer[RecapQuery, U, V]
object FutureDependencyTransformer
extends configapi.FutureDependencyTransformerFunctions[RecapQuery]
type DependencyTransformer[-U, +V] = configapi.DependencyTransformer[RecapQuery, U, V]
object DependencyTransformer extends configapi.DependencyTransformerFunctions[RecapQuery]
}

9
timelineranker/server/src/main/scala/com/twitter/timelineranker/decider/BUILD Normal file
View File

@ -0,0 +1,9 @@
scala_library(
sources = ["*.scala"],
platform = "java8",
tags = ["bazel-compatible"],
dependencies = [
"servo/decider",
"timelineranker/server/config",
],
)

83
timelineranker/server/src/main/scala/com/twitter/timelineranker/decider/DeciderKey.scala Normal file
View File

@ -0,0 +1,83 @@
package com.twitter.timelineranker.decider
import com.twitter.servo.decider.DeciderKeyEnum
object DeciderKey extends DeciderKeyEnum {
// Deciders that can be used to control load on TLR or its backends.
val EnableMaxConcurrencyLimiting: Value = Value("enable_max_concurrency_limiting")
// Deciders related to testing / debugging.
val EnableRoutingToRankerDevProxy: Value = Value("enable_routing_to_ranker_dev_proxy")
// Deciders related to authorization.
val ClientRequestAuthorization: Value = Value("client_request_authorization")
val ClientWriteWhitelist: Value = Value("client_write_whitelist")
val AllowTimelineMixerRecapProd: Value = Value("allow_timeline_mixer_recap_prod")
val AllowTimelineMixerRecycledProd: Value = Value("allow_timeline_mixer_recycled_prod")
val AllowTimelineMixerHydrateProd: Value = Value("allow_timeline_mixer_hydrate_prod")
val AllowTimelineMixerHydrateRecosProd: Value = Value("allow_timeline_mixer_hydrate_recos_prod")
val AllowTimelineMixerSeedAuthorsProd: Value = Value("allow_timeline_mixer_seed_authors_prod")
val AllowTimelineMixerSimclusterProd: Value = Value("allow_timeline_mixer_simcluster_prod")
val AllowTimelineMixerEntityTweetsProd: Value = Value("allow_timeline_mixer_entity_tweets_prod")
val AllowTimelineMixerListProd: Value = Value("allow_timeline_mixer_list_prod")
val AllowTimelineMixerListTweetProd: Value = Value("allow_timeline_mixer_list_tweet_prod")
val AllowTimelineMixerCommunityProd: Value = Value("allow_timeline_mixer_community_prod")
val AllowTimelineMixerCommunityTweetProd: Value = Value(
"allow_timeline_mixer_community_tweet_prod")
val AllowTimelineScorerRecommendedTrendTweetProd: Value = Value(
"allow_timeline_scorer_recommended_trend_tweet_prod")
val AllowTimelineMixerUtegLikedByTweetsProd: Value = Value(
"allow_timeline_mixer_uteg_liked_by_tweets_prod")
val AllowTimelineMixerStaging: Value = Value("allow_timeline_mixer_staging")
val AllowTimelineRankerProxy: Value = Value("allow_timeline_ranker_proxy")
val AllowTimelineRankerWarmup: Value = Value("allow_timeline_ranker_warmup")
val AllowTimelineScorerRecTopicTweetsProd: Value =
Value("allow_timeline_scorer_rec_topic_tweets_prod")
val AllowTimelineScorerPopularTopicTweetsProd: Value =
Value("allow_timeline_scorer_popular_topic_tweets_prod")
val AllowTimelineScorerHydrateTweetScoringProd: Value = Value(
"allow_timelinescorer_hydrate_tweet_scoring_prod")
val AllowTimelineServiceProd: Value = Value("allow_timeline_service_prod")
val AllowTimelineServiceStaging: Value = Value("allow_timeline_service_staging")
val RateLimitOverrideUnknown: Value = Value("rate_limit_override_unknown")
// Deciders related to reverse-chron home timeline materialization.
val MultiplierOfMaterializationTweetsFetched: Value = Value(
"multiplier_of_materialization_tweets_fetched"
)
val BackfillFilteredEntries: Value = Value("enable_backfill_filtered_entries")
val TweetsFilteringLossageThreshold: Value = Value("tweets_filtering_lossage_threshold")
val TweetsFilteringLossageLimit: Value = Value("tweets_filtering_lossage_limit")
val SupplementFollowsWithRealGraph: Value = Value("supplement_follows_with_real_graph")
// Deciders related to recap.
val RecapEnableContentFeaturesHydration: Value = Value("recap_enable_content_features_hydration")
val RecapMaxCountMultiplier: Value = Value("recap_max_count_multiplier")
val RecapEnableExtraSortingInResults: Value = Value("recap_enable_extra_sorting_in_results")
// Deciders related to recycled tweets.
val RecycledMaxCountMultiplier: Value = Value("recycled_max_count_multiplier")
val RecycledEnableContentFeaturesHydration: Value = Value(
"recycled_enable_content_features_hydration")
// Deciders related to entity tweets.
val EntityTweetsEnableContentFeaturesHydration: Value = Value(
"entity_tweets_enable_content_features_hydration")
// Deciders related to both recap and recycled tweets
val EnableRealGraphUsers: Value = Value("enable_real_graph_users")
val MaxRealGraphAndFollowedUsers: Value = Value("max_real_graph_and_followed_users")
// Deciders related to recap author
val RecapAuthorEnableNewPipeline: Value = Value("recap_author_enable_new_pipeline")
val RecapAuthorEnableContentFeaturesHydration: Value = Value(
"recap_author_enable_content_features_hydration")
// Deciders related to recap hydration (rectweet and ranked organic).
val RecapHydrationEnableContentFeaturesHydration: Value = Value(
"recap_hydration_enable_content_features_hydration")
// Deciders related to uteg liked by tweets
val UtegLikedByTweetsEnableContentFeaturesHydration: Value = Value(
"uteg_liked_by_tweets_enable_content_features_hydration")
}

39
timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/BUILD.bazel Normal file
View File

@ -0,0 +1,39 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"3rdparty/jvm/com/twitter/storehaus:core",
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
"configapi/configapi-decider/src/main/scala/com/twitter/timelines/configapi/decider",
"finagle/finagle-core/src/main",
"servo/util/src/main/scala",
"src/thrift/com/twitter/search:earlybird-scala",
"timelineranker/common/src/main/scala/com/twitter/timelineranker/model",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/common",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/config",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/core",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/repository",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/util",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/visibility",
"timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck",
"timelines/src/main/scala/com/twitter/timelines/clients/manhattan",
"timelines/src/main/scala/com/twitter/timelines/clients/relevance_search",
"timelines/src/main/scala/com/twitter/timelines/clients/tweetypie",
"timelines/src/main/scala/com/twitter/timelines/common/model",
"timelines/src/main/scala/com/twitter/timelines/config",
"timelines/src/main/scala/com/twitter/timelines/earlybird/common/options",
"timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils",
"timelines/src/main/scala/com/twitter/timelines/util",
"timelines/src/main/scala/com/twitter/timelines/util/stats",
"timelines/src/main/scala/com/twitter/timelines/visibility",
"timelines/src/main/scala/com/twitter/timelines/visibility/model",
"util/util-core:util-core-util",
"util/util-core/src/main/scala/com/twitter/conversions",
"util/util-stats/src/main/scala",
],
)

20
timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsRepository.scala Normal file
View File

@ -0,0 +1,20 @@
package com.twitter.timelineranker.entity_tweets
import com.twitter.timelineranker.model.CandidateTweetsResult
import com.twitter.timelineranker.model.RecapQuery
import com.twitter.util.Future
/**
* A repository of entity tweets candidates.
*
* For now, it does not cache any results therefore forwards all calls to the underlying source.
*/
class EntityTweetsRepository(source: EntityTweetsSource) {
def get(query: RecapQuery): Future[CandidateTweetsResult] = {
source.get(query)
}
def get(queries: Seq[RecapQuery]): Future[Seq[CandidateTweetsResult]] = {
source.get(queries)
}
}

60
timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsRepositoryBuilder.scala Normal file
View File

@ -0,0 +1,60 @@
package com.twitter.timelineranker.entity_tweets
import com.twitter.conversions.DurationOps._
import com.twitter.timelineranker.config.RequestScopes
import com.twitter.timelineranker.config.RuntimeConfiguration
import com.twitter.timelineranker.parameters.ConfigBuilder
import com.twitter.timelineranker.repository.CandidatesRepositoryBuilder
import com.twitter.timelineranker.visibility.SgsFollowGraphDataFields
import com.twitter.search.earlybird.thriftscala.EarlybirdService
import com.twitter.timelines.util.stats.RequestScope
import com.twitter.util.Duration
class EntityTweetsRepositoryBuilder(config: RuntimeConfiguration, configBuilder: ConfigBuilder)
extends CandidatesRepositoryBuilder(config) {
// Default client id for this repository if the upstream requests doesn't indicate one.
override val clientSubId = "community"
override val requestScope: RequestScope = RequestScopes.EntityTweetsSource
override val followGraphDataFieldsToFetch: SgsFollowGraphDataFields.ValueSet =
SgsFollowGraphDataFields.ValueSet(
SgsFollowGraphDataFields.FollowedUserIds,
SgsFollowGraphDataFields.MutuallyFollowingUserIds,
SgsFollowGraphDataFields.MutedUserIds
)
/**
* [1] timeout is derived from p9999 TLR <-> Earlybird latency and shall be less than
* request timeout of timelineranker client within downstream timelinemixer, which is
* 1s now
*
* [2] processing timeout is less than request timeout [1] with 100ms space for networking and
* other times such as gc
*/
override val searchProcessingTimeout: Duration = 550.milliseconds // [2]
override def earlybirdClient(scope: String): EarlybirdService.MethodPerEndpoint =
config.underlyingClients.createEarlybirdClient(
scope = scope,
requestTimeout = 650.milliseconds, // [1]
timeout = 900.milliseconds, // [1]
retryPolicy = config.underlyingClients.DefaultRetryPolicy
)
def apply(): EntityTweetsRepository = {
val entityTweetsSource = new EntityTweetsSource(
gizmoduckClient,
searchClient,
tweetyPieHighQoSClient,
userMetadataClient,
followGraphDataProvider,
clientFactories.visibilityEnforcerFactory.apply(
VisibilityRules,
RequestScopes.EntityTweetsSource
),
config.underlyingClients.contentFeaturesCache,
config.statsReceiver
)
new EntityTweetsRepository(entityTweetsSource)
}
}

71
timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsSearchResultsTransform.scala Normal file
View File

@ -0,0 +1,71 @@
package com.twitter.timelineranker.entity_tweets
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.core.CandidateEnvelope
import com.twitter.timelineranker.model.TweetIdRange
import com.twitter.timelines.clients.relevance_search.SearchClient
import com.twitter.timelines.clients.relevance_search.SearchClient.TweetTypes
import com.twitter.timelines.model.TweetId
import com.twitter.util.Future
object EntityTweetsSearchResultsTransform {
// If EntityTweetsQuery.maxCount is not specified, the following count is used.
val DefaultEntityTweetsMaxTweetCount = 200
}
/**
* Fetch entity tweets search results using the search client
* and populate them into the CandidateEnvelope
*/
class EntityTweetsSearchResultsTransform(
searchClient: SearchClient,
statsReceiver: StatsReceiver,
logSearchDebugInfo: Boolean = false)
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
import EntityTweetsSearchResultsTransform._
private[this] val maxCountStat = statsReceiver.stat("maxCount")
private[this] val numResultsFromSearchStat = statsReceiver.stat("numResultsFromSearch")
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
val maxCount = envelope.query.maxCount.getOrElse(DefaultEntityTweetsMaxTweetCount)
maxCountStat.add(maxCount)
val tweetIdRange = envelope.query.range
.map(TweetIdRange.fromTimelineRange)
.getOrElse(TweetIdRange.default)
val beforeTweetIdExclusive = tweetIdRange.toId
val afterTweetIdExclusive = tweetIdRange.fromId
val excludedTweetIds = envelope.query.excludedTweetIds.getOrElse(Seq.empty[TweetId]).toSet
val languages = envelope.query.languages.map(_.map(_.language))
envelope.followGraphData.inNetworkUserIdsFuture.flatMap { inNetworkUserIds =>
searchClient
.getEntityTweets(
userId = Some(envelope.query.userId),
followedUserIds = inNetworkUserIds.toSet,
maxCount = maxCount,
beforeTweetIdExclusive = beforeTweetIdExclusive,
afterTweetIdExclusive = afterTweetIdExclusive,
earlybirdOptions = envelope.query.earlybirdOptions,
semanticCoreIds = envelope.query.semanticCoreIds,
hashtags = envelope.query.hashtags,
languages = languages,
tweetTypes = TweetTypes.fromTweetKindOption(envelope.query.options),
searchOperator = envelope.query.searchOperator,
excludedTweetIds = excludedTweetIds,
logSearchDebugInfo = logSearchDebugInfo,
includeNullcastTweets = envelope.query.includeNullcastTweets.getOrElse(false),
includeTweetsFromArchiveIndex =
envelope.query.includeTweetsFromArchiveIndex.getOrElse(false),
authorIds = envelope.query.authorIds.map(_.toSet)
).map { results =>
numResultsFromSearchStat.add(results.size)
envelope.copy(searchResults = results)
}
}
}
}

146
timelineranker/server/src/main/scala/com/twitter/timelineranker/entity_tweets/EntityTweetsSource.scala Normal file
View File

@ -0,0 +1,146 @@
package com.twitter.timelineranker.entity_tweets
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.servo.util.FutureArrow
import com.twitter.storehaus.Store
import com.twitter.timelineranker.common._
import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
import com.twitter.timelineranker.model._
import com.twitter.timelineranker.parameters.entity_tweets.EntityTweetsParams._
import com.twitter.timelineranker.recap.model.ContentFeatures
import com.twitter.timelineranker.util.CopyContentFeaturesIntoHydratedTweetsTransform
import com.twitter.timelineranker.util.CopyContentFeaturesIntoThriftTweetFeaturesTransform
import com.twitter.timelineranker.util.TweetFilters
import com.twitter.timelineranker.visibility.FollowGraphDataProvider
import com.twitter.timelines.clients.gizmoduck.GizmoduckClient
import com.twitter.timelines.clients.manhattan.UserMetadataClient
import com.twitter.timelines.clients.relevance_search.SearchClient
import com.twitter.timelines.clients.tweetypie.TweetyPieClient
import com.twitter.timelines.model.TweetId
import com.twitter.timelines.util.FailOpenHandler
import com.twitter.timelines.util.stats.RequestStatsReceiver
import com.twitter.timelines.visibility.VisibilityEnforcer
import com.twitter.util.Future
class EntityTweetsSource(
gizmoduckClient: GizmoduckClient,
searchClient: SearchClient,
tweetyPieClient: TweetyPieClient,
userMetadataClient: UserMetadataClient,
followGraphDataProvider: FollowGraphDataProvider,
visibilityEnforcer: VisibilityEnforcer,
contentFeaturesCache: Store[TweetId, ContentFeatures],
statsReceiver: StatsReceiver) {
private[this] val baseScope = statsReceiver.scope("entityTweetsSource")
private[this] val requestStats = RequestStatsReceiver(baseScope)
private[this] val failOpenScope = baseScope.scope("failOpen")
private[this] val userProfileHandler = new FailOpenHandler(failOpenScope, "userProfileInfo")
private[this] val userLanguagesHandler = new FailOpenHandler(failOpenScope, "userLanguages")
private[this] val followGraphDataTransform = new FollowGraphDataTransform(
followGraphDataProvider = followGraphDataProvider,
maxFollowedUsersProvider = DependencyProvider.from(MaxFollowedUsersParam)
)
private[this] val fetchSearchResultsTransform = new EntityTweetsSearchResultsTransform(
searchClient = searchClient,
statsReceiver = baseScope
)
private[this] val userProfileInfoTransform =
new UserProfileInfoTransform(userProfileHandler, gizmoduckClient)
private[this] val languagesTransform =
new UserLanguagesTransform(userLanguagesHandler, userMetadataClient)
private[this] val visibilityEnforcingTransform = new VisibilityEnforcingTransform(
visibilityEnforcer
)
private[this] val filters = TweetFilters.ValueSet(
TweetFilters.DuplicateTweets,
TweetFilters.DuplicateRetweets
)
private[this] val hydratedTweetsFilter = new HydratedTweetsFilterTransform(
outerFilters = filters,
innerFilters = TweetFilters.None,
useFollowGraphData = false,
useSourceTweets = false,
statsReceiver = baseScope,
numRetweetsAllowed = HydratedTweetsFilterTransform.NumDuplicateRetweetsAllowed
)
private[this] val contentFeaturesHydrationTransform =
new ContentFeaturesHydrationTransformBuilder(
tweetyPieClient = tweetyPieClient,
contentFeaturesCache = contentFeaturesCache,
enableContentFeaturesGate = RecapQuery.paramGate(EnableContentFeaturesHydrationParam),
enableTokensInContentFeaturesGate =
RecapQuery.paramGate(EnableTokensInContentFeaturesHydrationParam),
enableTweetTextInContentFeaturesGate =
RecapQuery.paramGate(EnableTweetTextInContentFeaturesHydrationParam),
enableConversationControlContentFeaturesGate =
RecapQuery.paramGate(EnableConversationControlInContentFeaturesHydrationParam),
enableTweetMediaHydrationGate = RecapQuery.paramGate(EnableTweetMediaHydrationParam),
hydrateInReplyToTweets = false,
statsReceiver = baseScope
).build()
private[this] def hydratesContentFeatures(
hydratedEnvelope: HydratedCandidatesAndFeaturesEnvelope
): Boolean =
hydratedEnvelope.candidateEnvelope.query.hydratesContentFeatures.getOrElse(true)
private[this] val contentFeaturesTransformer = FutureArrow.choose(
predicate = hydratesContentFeatures,
ifTrue = contentFeaturesHydrationTransform
.andThen(CopyContentFeaturesIntoThriftTweetFeaturesTransform)
.andThen(CopyContentFeaturesIntoHydratedTweetsTransform),
ifFalse = FutureArrow[
HydratedCandidatesAndFeaturesEnvelope,
HydratedCandidatesAndFeaturesEnvelope
](Future.value) // empty transformer
)
private[this] val candidateGenerationTransform = new CandidateGenerationTransform(baseScope)
private[this] val hydrationAndFilteringPipeline =
CreateCandidateEnvelopeTransform
.andThen(followGraphDataTransform) // Fetch follow graph data
.andThen(fetchSearchResultsTransform) // fetch search results
.andThen(SearchResultDedupAndSortingTransform) // dedup and order search results
.andThen(CandidateTweetHydrationTransform) // hydrate search results
.andThen(visibilityEnforcingTransform) // filter hydrated tweets to visible ones
.andThen(hydratedTweetsFilter) // filter hydrated tweets based on predefined filter
.andThen(
TrimToMatchHydratedTweetsTransform
) // trim search result set to match filtered hydrated tweets (this needs to be accurate for feature hydration)
// runs the main pipeline in parallel with fetching user profile info and user languages
private[this] val featureHydrationDataTransform =
new FeatureHydrationDataTransform(
hydrationAndFilteringPipeline,
languagesTransform,
userProfileInfoTransform
)
private[this] val tweetFeaturesHydrationTransform =
OutOfNetworkTweetsSearchFeaturesHydrationTransform
.andThen(contentFeaturesTransformer)
private[this] val featureHydrationPipeline =
featureHydrationDataTransform
.andThen(tweetFeaturesHydrationTransform)
.andThen(candidateGenerationTransform)
def get(query: RecapQuery): Future[CandidateTweetsResult] = {
requestStats.addEventStats {
featureHydrationPipeline(query)
}
}
def get(queries: Seq[RecapQuery]): Future[Seq[CandidateTweetsResult]] = {
Future.collect(queries.map(get))
}
}

41
timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/BUILD Normal file
View File

@ -0,0 +1,41 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"3rdparty/jvm/com/twitter/storehaus:core",
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
"configapi/configapi-decider/src/main/scala/com/twitter/timelines/configapi/decider",
"finagle/finagle-core/src/main",
"servo/util/src/main/scala",
"src/thrift/com/twitter/search:earlybird-scala",
"timelineranker/common/src/main/scala/com/twitter/timelineranker/model",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/common",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/config",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/core",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/monitoring",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/repository",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/util",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/visibility",
"timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck",
"timelines/src/main/scala/com/twitter/timelines/clients/manhattan",
"timelines/src/main/scala/com/twitter/timelines/clients/relevance_search",
"timelines/src/main/scala/com/twitter/timelines/clients/tweetypie",
"timelines/src/main/scala/com/twitter/timelines/config",
"timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils",
"timelines/src/main/scala/com/twitter/timelines/model/tweet",
"timelines/src/main/scala/com/twitter/timelines/util",
"timelines/src/main/scala/com/twitter/timelines/util/stats",
"timelines/src/main/scala/com/twitter/timelines/visibility",
"timelines/src/main/scala/com/twitter/timelines/visibility/model",
"util/util-core:util-core-util",
"util/util-core/src/main/scala/com/twitter/conversions",
"util/util-stats/src/main/scala",
],
)

31
timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/InNetworkTweetRepository.scala Normal file
View File

@ -0,0 +1,31 @@
package com.twitter.timelineranker.in_network_tweets
import com.twitter.timelineranker.model.CandidateTweetsResult
import com.twitter.timelineranker.model.RecapQuery
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
import com.twitter.timelineranker.parameters.in_network_tweets.InNetworkTweetParams
import com.twitter.util.Future
/**
* A repository of in-network tweet candidates.
* For now, it does not cache any results therefore forwards all calls to the underlying source.
*/
class InNetworkTweetRepository(
source: InNetworkTweetSource,
realtimeCGSource: InNetworkTweetSource) {
private[this] val enableRealtimeCGProvider =
DependencyProvider.from(InNetworkTweetParams.EnableEarlybirdRealtimeCgMigrationParam)
def get(query: RecapQuery): Future[CandidateTweetsResult] = {
if (enableRealtimeCGProvider(query)) {
realtimeCGSource.get(query)
} else {
source.get(query)
}
}
def get(queries: Seq[RecapQuery]): Future[Seq[CandidateTweetsResult]] = {
Future.collect(queries.map(query => get(query)))
}
}

109
timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/InNetworkTweetRepositoryBuilder.scala Normal file
View File

@ -0,0 +1,109 @@
package com.twitter.timelineranker.in_network_tweets
import com.twitter.conversions.DurationOps._
import com.twitter.finagle.service.RetryPolicy
import com.twitter.search.earlybird.thriftscala.EarlybirdService
import com.twitter.timelineranker.config.RequestScopes
import com.twitter.timelineranker.config.RuntimeConfiguration
import com.twitter.timelineranker.parameters.ConfigBuilder
import com.twitter.timelineranker.repository.CandidatesRepositoryBuilder
import com.twitter.timelineranker.visibility.SgsFollowGraphDataFields
import com.twitter.timelines.util.stats.RequestScope
import com.twitter.timelines.visibility.model.CheckedUserActorType
import com.twitter.timelines.visibility.model.ExclusionReason
import com.twitter.timelines.visibility.model.VisibilityCheckStatus
import com.twitter.timelines.visibility.model.VisibilityCheckUser
import com.twitter.util.Duration
object InNetworkTweetRepositoryBuilder {
val VisibilityRuleExclusions: Set[ExclusionReason] = Set[ExclusionReason](
ExclusionReason(
CheckedUserActorType(Some(false), VisibilityCheckUser.SourceUser),
Set(VisibilityCheckStatus.Blocked)
)
)
private val EarlybirdTimeout = 600.milliseconds
private val EarlybirdRequestTimeout = 600.milliseconds
/**
* The timeouts below are only used for the Earlybird Cluster Migration
*/
private val EarlybirdRealtimeCGTimeout = 600.milliseconds
private val EarlybirdRealtimeCGRequestTimeout = 600.milliseconds
}
class InNetworkTweetRepositoryBuilder(config: RuntimeConfiguration, configBuilder: ConfigBuilder)
extends CandidatesRepositoryBuilder(config) {
import InNetworkTweetRepositoryBuilder._
override val clientSubId = "recycled_tweets"
override val requestScope: RequestScope = RequestScopes.InNetworkTweetSource
override val followGraphDataFieldsToFetch: SgsFollowGraphDataFields.ValueSet =
SgsFollowGraphDataFields.ValueSet(
SgsFollowGraphDataFields.FollowedUserIds,
SgsFollowGraphDataFields.MutuallyFollowingUserIds,
SgsFollowGraphDataFields.MutedUserIds,
SgsFollowGraphDataFields.RetweetsMutedUserIds
)
override val searchProcessingTimeout: Duration = 200.milliseconds
override def earlybirdClient(scope: String): EarlybirdService.MethodPerEndpoint =
config.underlyingClients.createEarlybirdClient(
scope = scope,
requestTimeout = EarlybirdRequestTimeout,
timeout = EarlybirdTimeout,
retryPolicy = RetryPolicy.Never
)
private lazy val searchClientForSourceTweets =
newSearchClient(clientId = clientSubId + "_source_tweets")
/** The RealtimeCG clients below are only used for the Earlybird Cluster Migration */
private def earlybirdRealtimeCGClient(scope: String): EarlybirdService.MethodPerEndpoint =
config.underlyingClients.createEarlybirdRealtimeCgClient(
scope = scope,
requestTimeout = EarlybirdRealtimeCGRequestTimeout,
timeout = EarlybirdRealtimeCGTimeout,
retryPolicy = RetryPolicy.Never
)
private val realtimeCGClientSubId = "realtime_cg_recycled_tweets"
private lazy val searchRealtimeCGClient =
newSearchClient(earlybirdRealtimeCGClient, clientId = realtimeCGClientSubId)
def apply(): InNetworkTweetRepository = {
val inNetworkTweetSource = new InNetworkTweetSource(
gizmoduckClient,
searchClient,
searchClientForSourceTweets,
tweetyPieHighQoSClient,
userMetadataClient,
followGraphDataProvider,
config.underlyingClients.contentFeaturesCache,
clientFactories.visibilityEnforcerFactory.apply(
VisibilityRules,
RequestScopes.InNetworkTweetSource,
reasonsToExclude = InNetworkTweetRepositoryBuilder.VisibilityRuleExclusions
),
config.statsReceiver
)
val inNetworkTweetRealtimeCGSource = new InNetworkTweetSource(
gizmoduckClient,
searchRealtimeCGClient,
searchClientForSourceTweets, // do not migrate source_tweets as they are sharded by TweetID
tweetyPieHighQoSClient,
userMetadataClient,
followGraphDataProvider,
config.underlyingClients.contentFeaturesCache,
clientFactories.visibilityEnforcerFactory.apply(
VisibilityRules,
RequestScopes.InNetworkTweetSource,
reasonsToExclude = InNetworkTweetRepositoryBuilder.VisibilityRuleExclusions
),
config.statsReceiver.scope("replacementRealtimeCG")
)
new InNetworkTweetRepository(inNetworkTweetSource, inNetworkTweetRealtimeCGSource)
}
}

271
timelineranker/server/src/main/scala/com/twitter/timelineranker/in_network_tweets/InNetworkTweetSource.scala Normal file
View File

@ -0,0 +1,271 @@
package com.twitter.timelineranker.in_network_tweets
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.servo.util.FutureArrow
import com.twitter.storehaus.Store
import com.twitter.timelineranker.common._
import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
import com.twitter.timelineranker.model._
import com.twitter.timelineranker.monitoring.UsersSearchResultMonitoringTransform
import com.twitter.timelineranker.parameters.in_network_tweets.InNetworkTweetParams
import com.twitter.timelineranker.parameters.monitoring.MonitoringParams
import com.twitter.timelineranker.parameters.recap.RecapParams
import com.twitter.timelineranker.recap.model.ContentFeatures
import com.twitter.timelineranker.util.CopyContentFeaturesIntoHydratedTweetsTransform
import com.twitter.timelineranker.util.CopyContentFeaturesIntoThriftTweetFeaturesTransform
import com.twitter.timelineranker.util.TweetFilters
import com.twitter.timelineranker.visibility.FollowGraphDataProvider
import com.twitter.timelines.clients.gizmoduck.GizmoduckClient
import com.twitter.timelines.clients.manhattan.UserMetadataClient
import com.twitter.timelines.clients.relevance_search.SearchClient
import com.twitter.timelines.clients.tweetypie.TweetyPieClient
import com.twitter.timelines.model.TweetId
import com.twitter.timelines.util.FailOpenHandler
import com.twitter.timelines.util.stats.RequestStatsReceiver
import com.twitter.timelines.visibility.VisibilityEnforcer
import com.twitter.util.Future
class InNetworkTweetSource(
gizmoduckClient: GizmoduckClient,
searchClient: SearchClient,
searchClientForSourceTweets: SearchClient,
tweetyPieClient: TweetyPieClient,
userMetadataClient: UserMetadataClient,
followGraphDataProvider: FollowGraphDataProvider,
contentFeaturesCache: Store[TweetId, ContentFeatures],
visibilityEnforcer: VisibilityEnforcer,
statsReceiver: StatsReceiver) {
private[this] val baseScope = statsReceiver.scope("recycledTweetSource")
private[this] val requestStats = RequestStatsReceiver(baseScope)
private[this] val failOpenScope = baseScope.scope("failOpen")
private[this] val userProfileHandler = new FailOpenHandler(failOpenScope, "userProfileInfo")
private[this] val userLanguagesHandler = new FailOpenHandler(failOpenScope, "userLanguages")
private[this] val sourceTweetSearchHandler =
new FailOpenHandler(failOpenScope, "sourceTweetSearch")
private[this] val filters = TweetFilters.ValueSet(
TweetFilters.DuplicateTweets,
TweetFilters.DuplicateRetweets,
TweetFilters.TweetsFromNotFollowedUsers,
TweetFilters.NonReplyDirectedAtNotFollowedUsers
)
private[this] val hydrateReplyRootTweetProvider =
DependencyProvider.from(InNetworkTweetParams.EnableReplyRootTweetHydrationParam)
private[this] val sourceTweetsSearchResultsTransform = new SourceTweetsSearchResultsTransform(
searchClientForSourceTweets,
sourceTweetSearchHandler,
hydrateReplyRootTweetProvider = hydrateReplyRootTweetProvider,
perRequestSourceSearchClientIdProvider = DependencyProvider.None,
baseScope
)
private[this] val visibilityEnforcingTransform = new VisibilityEnforcingTransform(
visibilityEnforcer
)
private[this] val hydratedTweetsFilter = new HydratedTweetsFilterTransform(
outerFilters = filters,
innerFilters = TweetFilters.None,
useFollowGraphData = true,
useSourceTweets = true,
statsReceiver = baseScope,
numRetweetsAllowed = HydratedTweetsFilterTransform.NumDuplicateRetweetsAllowed
)
private[this] val dynamicHydratedTweetsFilter = new TweetKindOptionHydratedTweetsFilterTransform(
useFollowGraphData = true,
useSourceTweets = true,
statsReceiver = baseScope
)
private[this] val userProfileInfoTransform =
new UserProfileInfoTransform(userProfileHandler, gizmoduckClient)
private[this] val languagesTransform =
new UserLanguagesTransform(userLanguagesHandler, userMetadataClient)
private[this] def hydratesContentFeatures(
hydratedEnvelope: HydratedCandidatesAndFeaturesEnvelope
): Boolean =
hydratedEnvelope.candidateEnvelope.query.hydratesContentFeatures.getOrElse(true)
private[this] val contentFeaturesTransformer = FutureArrow.choose(
predicate = hydratesContentFeatures,
ifTrue = contentFeaturesHydrationTransform
.andThen(CopyContentFeaturesIntoThriftTweetFeaturesTransform)
.andThen(CopyContentFeaturesIntoHydratedTweetsTransform),
ifFalse = FutureArrow[
HydratedCandidatesAndFeaturesEnvelope,
HydratedCandidatesAndFeaturesEnvelope
](Future.value) // empty transformer
)
private[this] val contentFeaturesHydrationTransform =
new ContentFeaturesHydrationTransformBuilder(
tweetyPieClient = tweetyPieClient,
contentFeaturesCache = contentFeaturesCache,
enableContentFeaturesGate =
RecapQuery.paramGate(InNetworkTweetParams.EnableContentFeaturesHydrationParam),
enableTokensInContentFeaturesGate =
RecapQuery.paramGate(InNetworkTweetParams.EnableTokensInContentFeaturesHydrationParam),
enableTweetTextInContentFeaturesGate =
RecapQuery.paramGate(InNetworkTweetParams.EnableTweetTextInContentFeaturesHydrationParam),
enableConversationControlContentFeaturesGate = RecapQuery.paramGate(
InNetworkTweetParams.EnableConversationControlInContentFeaturesHydrationParam),
enableTweetMediaHydrationGate = RecapQuery.paramGate(
InNetworkTweetParams.EnableTweetMediaHydrationParam
),
hydrateInReplyToTweets = true,
statsReceiver = baseScope
).build()
private[this] val candidateGenerationTransform = new CandidateGenerationTransform(baseScope)
private[this] val maxFollowedUsersProvider =
DependencyProvider.from(InNetworkTweetParams.MaxFollowedUsersParam)
private[this] val earlybirdReturnAllResultsProvider =
DependencyProvider.from(InNetworkTweetParams.EnableEarlybirdReturnAllResultsParam)
private[this] val relevanceOptionsMaxHitsToProcessProvider =
DependencyProvider.from(InNetworkTweetParams.RelevanceOptionsMaxHitsToProcessParam)
private[this] val followGraphDataTransform =
new FollowGraphDataTransform(followGraphDataProvider, maxFollowedUsersProvider)
private[this] val enableRealGraphUsersProvider =
DependencyProvider.from(RecapParams.EnableRealGraphUsersParam)
private[this] val maxRealGraphAndFollowedUsersProvider =
DependencyProvider.from(RecapParams.MaxRealGraphAndFollowedUsersParam)
private[this] val maxRealGraphAndFollowedUsersFSOverrideProvider =
DependencyProvider.from(RecapParams.MaxRealGraphAndFollowedUsersFSOverrideParam)
private[this] val imputeRealGraphAuthorWeightsProvider =
DependencyProvider.from(RecapParams.ImputeRealGraphAuthorWeightsParam)
private[this] val imputeRealGraphAuthorWeightsPercentileProvider =
DependencyProvider.from(RecapParams.ImputeRealGraphAuthorWeightsPercentileParam)
private[this] val maxRealGraphAndFollowedUsersFromDeciderAndFS = DependencyProvider { envelope =>
maxRealGraphAndFollowedUsersFSOverrideProvider(envelope).getOrElse(
maxRealGraphAndFollowedUsersProvider(envelope))
}
private[this] val followAndRealGraphCombiningTransform = new FollowAndRealGraphCombiningTransform(
followGraphDataProvider = followGraphDataProvider,
maxFollowedUsersProvider = maxFollowedUsersProvider,
enableRealGraphUsersProvider = enableRealGraphUsersProvider,
maxRealGraphAndFollowedUsersProvider = maxRealGraphAndFollowedUsersFromDeciderAndFS,
imputeRealGraphAuthorWeightsProvider = imputeRealGraphAuthorWeightsProvider,
imputeRealGraphAuthorWeightsPercentileProvider = imputeRealGraphAuthorWeightsPercentileProvider,
statsReceiver = baseScope
)
private[this] val maxCountProvider = DependencyProvider { query =>
query.maxCount.getOrElse(query.params(InNetworkTweetParams.DefaultMaxTweetCount))
}
private[this] val maxCountWithMarginProvider = DependencyProvider { query =>
val maxCount = query.maxCount.getOrElse(query.params(InNetworkTweetParams.DefaultMaxTweetCount))
val multiplier = query.params(InNetworkTweetParams.MaxCountMultiplierParam)
(maxCount * multiplier).toInt
}
private[this] val debugAuthorsMonitoringProvider =
DependencyProvider.from(MonitoringParams.DebugAuthorsAllowListParam)
private[this] val retrieveSearchResultsTransform = new RecapSearchResultsTransform(
searchClient = searchClient,
maxCountProvider = maxCountWithMarginProvider,
returnAllResultsProvider = earlybirdReturnAllResultsProvider,
relevanceOptionsMaxHitsToProcessProvider = relevanceOptionsMaxHitsToProcessProvider,
enableExcludeSourceTweetIdsProvider = DependencyProvider.True,
enableSettingTweetTypesWithTweetKindOptionProvider =
DependencyProvider.from(RecapParams.EnableSettingTweetTypesWithTweetKindOption),
perRequestSearchClientIdProvider = DependencyProvider.None,
statsReceiver = baseScope,
logSearchDebugInfo = false
)
private[this] val preTruncateSearchResultsTransform =
new UsersSearchResultMonitoringTransform(
name = "RecapSearchResultsTruncationTransform",
new RecapSearchResultsTruncationTransform(
extraSortBeforeTruncationGate = DependencyProvider.True,
maxCountProvider = maxCountWithMarginProvider,
statsReceiver = baseScope.scope("afterSearchResultsTransform")
),
baseScope.scope("afterSearchResultsTransform"),
debugAuthorsMonitoringProvider
)
private[this] val finalTruncationTransform = new UsersSearchResultMonitoringTransform(
name = "RecapSearchResultsTruncationTransform",
new RecapSearchResultsTruncationTransform(
extraSortBeforeTruncationGate = DependencyProvider.True,
maxCountProvider = maxCountProvider,
statsReceiver = baseScope.scope("finalTruncation")
),
baseScope.scope("finalTruncation"),
debugAuthorsMonitoringProvider
)
// Fetch source tweets based on search results present in the envelope
// and hydrate them.
private[this] val fetchAndHydrateSourceTweets =
sourceTweetsSearchResultsTransform
.andThen(SourceTweetHydrationTransform)
// Hydrate candidate tweets and fetch source tweets in parallel
private[this] val hydrateTweetsAndSourceTweetsInParallel =
new HydrateTweetsAndSourceTweetsInParallelTransform(
candidateTweetHydration = CandidateTweetHydrationTransform,
sourceTweetHydration = fetchAndHydrateSourceTweets
)
private[this] val trimToMatchSearchResultsTransform = new TrimToMatchSearchResultsTransform(
hydrateReplyRootTweetProvider = hydrateReplyRootTweetProvider,
statsReceiver = baseScope
)
private[this] val hydrationAndFilteringPipeline =
CreateCandidateEnvelopeTransform // Create empty CandidateEnvelope
.andThen(followGraphDataTransform) // Fetch follow graph data
.andThen(followAndRealGraphCombiningTransform) // Experiment: expand seed author set
.andThen(retrieveSearchResultsTransform) // Fetch search results
.andThen(
preTruncateSearchResultsTransform
) // truncate the search result up to maxCount + some margin, preserving the random tweet
.andThen(SearchResultDedupAndSortingTransform) // dedups, and sorts reverse-chron
.andThen(hydrateTweetsAndSourceTweetsInParallel) // candidates + source tweets in parallel
.andThen(visibilityEnforcingTransform) // filter hydrated tweets to visible ones
.andThen(hydratedTweetsFilter) // filter hydrated tweets based on predefined filter
.andThen(dynamicHydratedTweetsFilter) // filter hydrated tweets based on query TweetKindOption
.andThen(TrimToMatchHydratedTweetsTransform) // trim searchResult to match with hydratedTweets
.andThen(
finalTruncationTransform
) // truncate the searchResult to exactly up to maxCount, preserving the random tweet
.andThen(
trimToMatchSearchResultsTransform
) // trim other fields to match with the final searchResult
// runs the main pipeline in parallel with fetching user profile info and user languages
private[this] val featureHydrationDataTransform = new FeatureHydrationDataTransform(
hydrationAndFilteringPipeline,
languagesTransform,
userProfileInfoTransform
)
private[this] val featureHydrationPipeline =
featureHydrationDataTransform
.andThen(InNetworkTweetsSearchFeaturesHydrationTransform)
.andThen(contentFeaturesTransformer)
.andThen(candidateGenerationTransform)
def get(query: RecapQuery): Future[CandidateTweetsResult] = {
requestStats.addEventStats {
featureHydrationPipeline(query)
}
}
def get(queries: Seq[RecapQuery]): Future[Seq[CandidateTweetsResult]] = {
Future.collect(queries.map(get))
}
}

15
timelineranker/server/src/main/scala/com/twitter/timelineranker/monitoring/BUILD Normal file
View File

@ -0,0 +1,15 @@
scala_library(
sources = ["*.scala"],
platform = "java8",
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"3rdparty/jvm/org/apache/thrift:libthrift",
"scrooge/scrooge-core/src/main/scala",
"servo/util",
"src/thrift/com/twitter/search:earlybird-scala",
"timelineranker/common/src/main/scala/com/twitter/timelineranker/model",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/core",
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
],
)

52
timelineranker/server/src/main/scala/com/twitter/timelineranker/monitoring/UsersSearchResultMonitoringTransform.scala Normal file
View File

@ -0,0 +1,52 @@
package com.twitter.timelineranker.monitoring
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.search.earlybird.thriftscala.ThriftSearchResult
import com.twitter.servo.util.FutureArrow
import com.twitter.timelineranker.core.CandidateEnvelope
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
import com.twitter.util.Future
/**
* Captures tweet counts pre and post transformation for a list of users
*/
class UsersSearchResultMonitoringTransform(
name: String,
underlyingTransformer: FutureArrow[CandidateEnvelope, CandidateEnvelope],
statsReceiver: StatsReceiver,
debugAuthorListDependencyProvider: DependencyProvider[Seq[Long]])
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
private val scopedStatsReceiver = statsReceiver.scope(name)
private val preTransformCounter = (userId: Long) =>
scopedStatsReceiver
.scope("pre_transform").scope(userId.toString).counter("debug_author_list")
private val postTransformCounter = (userId: Long) =>
scopedStatsReceiver
.scope("post_transform").scope(userId.toString).counter("debug_author_list")
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
val debugAuthorList = debugAuthorListDependencyProvider.apply(envelope.query)
envelope.searchResults
.filter(isTweetFromDebugAuthorList(_, debugAuthorList))
.flatMap(_.metadata)
.foreach(metadata => preTransformCounter(metadata.fromUserId).incr())
underlyingTransformer
.apply(envelope)
.map { result =>
envelope.searchResults
.filter(isTweetFromDebugAuthorList(_, debugAuthorList))
.flatMap(_.metadata)
.foreach(metadata => postTransformCounter(metadata.fromUserId).incr())
result
}
}
private def isTweetFromDebugAuthorList(
searchResult: ThriftSearchResult,
debugAuthorList: Seq[Long]
): Boolean =
searchResult.metadata.exists(metadata => debugAuthorList.contains(metadata.fromUserId))
}

15
timelineranker/server/src/main/scala/com/twitter/timelineranker/observe/BUILD.bazel Normal file
View File

@ -0,0 +1,15 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"src/thrift/com/twitter/timelineranker/server/model:thrift-scala",
"timelineranker/common:model",
"timelines:observe",
"timelines/src/main/scala/com/twitter/timelines/authorization",
"timelines/src/main/scala/com/twitter/timelines/features",
"timelines/src/main/scala/com/twitter/timelines/model/types",
"util/util-core:util-core-util",
],
)

36
timelineranker/server/src/main/scala/com/twitter/timelineranker/observe/DebugObserverBuilder.scala Normal file
View File

@ -0,0 +1,36 @@
package com.twitter.timelineranker.observe
import com.twitter.servo.util.Gate
import com.twitter.timelineranker.model.TimelineQuery
import com.twitter.timelines.features.Features
import com.twitter.timelines.features.UserList
import com.twitter.timelines.observe.DebugObserver
import com.twitter.timelineranker.{thriftscala => thrift}
/**
* Builds the DebugObserver that is attached to thrift requests.
* This class exists to centralize the gates that determine whether or not
* to enable debug transcripts for a particular request.
*/
class DebugObserverBuilder(whitelist: UserList) {
lazy val observer: DebugObserver = build()
private[this] def build(): DebugObserver = {
new DebugObserver(queryGate)
}
private[observe] def queryGate: Gate[Any] = {
val shouldEnableDebug = whitelist.userIdGate(Features.DebugTranscript)
Gate { a: Any =>
a match {
case q: thrift.EngagedTweetsQuery => shouldEnableDebug(q.userId)
case q: thrift.RecapHydrationQuery => shouldEnableDebug(q.userId)
case q: thrift.RecapQuery => shouldEnableDebug(q.userId)
case q: TimelineQuery => shouldEnableDebug(q.userId)
case _ => false
}
}
}
}

35
timelineranker/server/src/main/scala/com/twitter/timelineranker/observe/ObservedRequests.scala Normal file
View File

@ -0,0 +1,35 @@
package com.twitter.timelineranker.observe
import com.twitter.timelines.authorization.ReadRequest
import com.twitter.timelines.model.UserId
import com.twitter.timelines.observe.ObservedAndValidatedRequests
import com.twitter.timelines.observe.ServiceObserver
import com.twitter.timelines.observe.ServiceTracer
import com.twitter.util.Future
trait ObservedRequests extends ObservedAndValidatedRequests {
def observeAndValidate[R, Q](
request: Q,
viewerIds: Seq[UserId],
stats: ServiceObserver.Stats[Q],
exceptionHandler: PartialFunction[Throwable, Future[R]]
)(
f: Q => Future[R]
): Future[R] = {
super.observeAndValidate[Q, R](
request,
viewerIds,
ReadRequest,
validateRequest,
exceptionHandler,
stats,
ServiceTracer.identity[Q]
)(f)
}
def validateRequest[Q](request: Q): Unit = {
// TimelineQuery and its derived classes do not permit invalid instances to be constructed.
// Therefore no additional validation is required.
}
}

18
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/BUILD Normal file
View File

@ -0,0 +1,18 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
tags = ["bazel-compatible"],
dependencies = [
"configapi/configapi-core",
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
"servo/decider",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets",
],
)

60
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/ConfigBuilder.scala Normal file
View File

@ -0,0 +1,60 @@
package com.twitter.timelineranker.parameters
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.servo.decider.DeciderGateBuilder
import com.twitter.timelineranker.parameters.entity_tweets.EntityTweetsProduction
import com.twitter.timelineranker.parameters.recap.RecapProduction
import com.twitter.timelineranker.parameters.recap_author.RecapAuthorProduction
import com.twitter.timelineranker.parameters.recap_hydration.RecapHydrationProduction
import com.twitter.timelineranker.parameters.in_network_tweets.InNetworkTweetProduction
import com.twitter.timelineranker.parameters.revchron.ReverseChronProduction
import com.twitter.timelineranker.parameters.uteg_liked_by_tweets.UtegLikedByTweetsProduction
import com.twitter.timelineranker.parameters.monitoring.MonitoringProduction
import com.twitter.timelines.configapi.CompositeConfig
import com.twitter.timelines.configapi.Config
/**
* Builds global composite config containing prioritized "layers" of parameter overrides
* based on whitelists, experiments, and deciders. Generated config can be used in tests with
* mocked decider and whitelist.
*/
class ConfigBuilder(deciderGateBuilder: DeciderGateBuilder, statsReceiver: StatsReceiver) {
/**
* Production config which includes all configs which contribute to production behavior. At
* minimum, it should include all configs containing decider-based param overrides.
*
* It is important that the production config include all production param overrides as it is
* used to build holdback experiment configs; If the production config doesn't include all param
* overrides supporting production behavior then holdback experiment "production" buckets will
* not reflect production behavior.
*/
val prodConfig: Config = new CompositeConfig(
Seq(
new RecapProduction(deciderGateBuilder, statsReceiver).config,
new InNetworkTweetProduction(deciderGateBuilder).config,
new ReverseChronProduction(deciderGateBuilder).config,
new EntityTweetsProduction(deciderGateBuilder).config,
new RecapAuthorProduction(deciderGateBuilder).config,
new RecapHydrationProduction(deciderGateBuilder).config,
new UtegLikedByTweetsProduction(deciderGateBuilder).config,
MonitoringProduction.config
),
"prodConfig"
)
val whitelistConfig: Config = new CompositeConfig(
Seq(
// No whitelists configured at present.
),
"whitelistConfig"
)
val rootConfig: Config = new CompositeConfig(
Seq(
whitelistConfig,
prodConfig
),
"rootConfig"
)
}

12
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets/BUILD Normal file
View File

@ -0,0 +1,12 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
"configapi/configapi-decider/src/main/scala/com/twitter/timelines/configapi/decider",
"servo/decider",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/decider",
],
)

65
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets/EntityTweetsParams.scala Normal file
View File

@ -0,0 +1,65 @@
package com.twitter.timelineranker.parameters.entity_tweets
import com.twitter.timelineranker.decider.DeciderKey
import com.twitter.timelines.configapi.decider.DeciderParam
import com.twitter.timelines.configapi.FSBoundedParam
import com.twitter.timelines.configapi.FSParam
object EntityTweetsParams {
/**
* Controls limit on the number of followed users fetched from SGS.
*/
object MaxFollowedUsersParam
extends FSBoundedParam[Int](
name = "entity_tweets_max_followed_users",
default = 1000,
min = 0,
max = 5000
)
/**
* Enables semantic core, penguin, and tweetypie content features in entity tweets source.
*/
object EnableContentFeaturesHydrationParam
extends DeciderParam[Boolean](
decider = DeciderKey.EntityTweetsEnableContentFeaturesHydration,
default = false
)
/**
* additionally enables tokens when hydrating content features.
*/
object EnableTokensInContentFeaturesHydrationParam
extends FSParam(
name = "entity_tweets_enable_tokens_in_content_features_hydration",
default = false
)
/**
* additionally enables tweet text when hydrating content features.
* This only works if EnableContentFeaturesHydrationParam is set to true
*/
object EnableTweetTextInContentFeaturesHydrationParam
extends FSParam(
name = "entity_tweets_enable_tweet_text_in_content_features_hydration",
default = false
)
/**
* additionally enables conversationControl when hydrating content features.
* This only works if EnableContentFeaturesHydrationParam is set to true
*/
object EnableConversationControlInContentFeaturesHydrationParam
extends FSParam(
name = "conversation_control_in_content_features_hydration_entity_enable",
default = false
)
object EnableTweetMediaHydrationParam
extends FSParam(
name = "tweet_media_hydration_entity_tweets_enable",
default = false
)
}

42
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets/EntityTweetsProduction.scala Normal file
View File

@ -0,0 +1,42 @@
package com.twitter.timelineranker.parameters.entity_tweets
import com.twitter.servo.decider.DeciderGateBuilder
import com.twitter.servo.decider.DeciderKeyName
import com.twitter.timelineranker.decider.DeciderKey
import com.twitter.timelineranker.parameters.entity_tweets.EntityTweetsParams._
import com.twitter.timelines.configapi.decider.DeciderUtils
import com.twitter.timelines.configapi.BaseConfig
import com.twitter.timelines.configapi.BaseConfigBuilder
import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil
import com.twitter.timelines.configapi.Param
object EntityTweetsProduction {
val deciderByParam: Map[Param[_], DeciderKeyName] = Map[Param[_], DeciderKeyName](
EnableContentFeaturesHydrationParam -> DeciderKey.EntityTweetsEnableContentFeaturesHydration
)
}
case class EntityTweetsProduction(deciderGateBuilder: DeciderGateBuilder) {
val booleanDeciderOverrides = DeciderUtils.getBooleanDeciderOverrides(
deciderGateBuilder,
EnableContentFeaturesHydrationParam
)
val booleanFeatureSwitchOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides(
EnableTokensInContentFeaturesHydrationParam,
EnableTweetTextInContentFeaturesHydrationParam,
EnableConversationControlInContentFeaturesHydrationParam,
EnableTweetMediaHydrationParam
)
val intFeatureSwitchOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides(
MaxFollowedUsersParam
)
val config: BaseConfig = new BaseConfigBuilder()
.set(booleanDeciderOverrides: _*)
.set(booleanFeatureSwitchOverrides: _*)
.set(intFeatureSwitchOverrides: _*)
.build()
}

15
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets/BUILD Normal file
View File

@ -0,0 +1,15 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
"configapi/configapi-decider/src/main/scala/com/twitter/timelines/configapi/decider",
"servo/decider",
"servo/util/src/main/scala",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/decider",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util",
],
)

133
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets/InNetworkTweetParams.scala Normal file
View File

@ -0,0 +1,133 @@
package com.twitter.timelineranker.parameters.in_network_tweets
import com.twitter.timelineranker.parameters.recap.RecapQueryContext
import com.twitter.timelines.configapi.decider._
import com.twitter.timelines.configapi.FSBoundedParam
import com.twitter.timelines.configapi.FSParam
import com.twitter.timelines.configapi.Param
object InNetworkTweetParams {
import RecapQueryContext._
/**
* Controls limit on the number of followed users fetched from SGS.
*
* The specific default value below is for blender-timelines parity.
*/
object MaxFollowedUsersParam
extends FSBoundedParam[Int](
name = "recycled_max_followed_users",
default = MaxFollowedUsers.default,
min = MaxFollowedUsers.bounds.minInclusive,
max = MaxFollowedUsers.bounds.maxInclusive
)
/**
* Controls limit on the number of hits for Earlybird.
*
*/
object RelevanceOptionsMaxHitsToProcessParam
extends FSBoundedParam[Int](
name = "recycled_relevance_options_max_hits_to_process",
default = 500,
min = 100,
max = 20000
)
/**
* Fallback value for maximum number of search results, if not specified by query.maxCount
*/
object DefaultMaxTweetCount extends Param(200)
/**
* We multiply maxCount (caller supplied value) by this multiplier and fetch those many
* candidates from search so that we are left with sufficient number of candidates after
* hydration and filtering.
*/
object MaxCountMultiplierParam
extends Param(MaxCountMultiplier.default)
with DeciderValueConverter[Double] {
override def convert: IntConverter[Double] =
OutputBoundIntConverter[Double](divideDeciderBy100 _, MaxCountMultiplier.bounds)
}
/**
* Enable [[SearchQueryBuilder.createExcludedSourceTweetIdsQuery]]
*/
object EnableExcludeSourceTweetIdsQueryParam
extends FSParam[Boolean](
name = "recycled_exclude_source_tweet_ids_query_enable",
default = false
)
object EnableEarlybirdReturnAllResultsParam
extends FSParam[Boolean](
name = "recycled_enable_earlybird_return_all_results",
default = true
)
/**
* FS-controlled param to enable anti-dilution transform for DDG-16198
*/
object RecycledMaxFollowedUsersEnableAntiDilutionParam
extends FSParam[Boolean](
name = "recycled_max_followed_users_enable_anti_dilution",
default = false
)
/**
* Enables semantic core, penguin, and tweetypie content features in recycled source.
*/
object EnableContentFeaturesHydrationParam extends Param(default = true)
/**
* additionally enables tokens when hydrating content features.
*/
object EnableTokensInContentFeaturesHydrationParam
extends FSParam(
name = "recycled_enable_tokens_in_content_features_hydration",
default = false
)
/**
* additionally enables tweet text when hydrating content features.
* This only works if EnableContentFeaturesHydrationParam is set to true
*/
object EnableTweetTextInContentFeaturesHydrationParam
extends FSParam(
name = "recycled_enable_tweet_text_in_content_features_hydration",
default = false
)
/**
* Enables hydrating root tweet of in-network replies and extended replies
*/
object EnableReplyRootTweetHydrationParam
extends FSParam(
name = "recycled_enable_reply_root_tweet_hydration",
default = true
)
/**
* additionally enables conversationControl when hydrating content features.
* This only works if EnableContentFeaturesHydrationParam is set to true
*/
object EnableConversationControlInContentFeaturesHydrationParam
extends FSParam(
name = "conversation_control_in_content_features_hydration_recycled_enable",
default = false
)
object EnableTweetMediaHydrationParam
extends FSParam(
name = "tweet_media_hydration_recycled_enable",
default = false
)
object EnableEarlybirdRealtimeCgMigrationParam
extends FSParam(
name = "recycled_enable_earlybird_realtime_cg_migration",
default = false
)
}

71
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets/InNetworkTweetProduction.scala Normal file
View File

@ -0,0 +1,71 @@
package com.twitter.timelineranker.parameters.in_network_tweets
import com.twitter.servo.decider.DeciderGateBuilder
import com.twitter.servo.decider.DeciderKeyName
import com.twitter.timelineranker.decider.DeciderKey
import com.twitter.timelineranker.parameters.in_network_tweets.InNetworkTweetParams._
import com.twitter.timelineranker.parameters.util.ConfigHelper
import com.twitter.timelines.configapi._
import com.twitter.servo.decider.DeciderKeyEnum
object InNetworkTweetProduction {
val deciderByParam: Map[Param[_], DeciderKeyEnum#Value] = Map[Param[_], DeciderKeyName](
EnableContentFeaturesHydrationParam -> DeciderKey.RecycledEnableContentFeaturesHydration,
MaxCountMultiplierParam -> DeciderKey.RecycledMaxCountMultiplier
)
val doubleParams: Seq[MaxCountMultiplierParam.type] = Seq(
MaxCountMultiplierParam
)
val booleanDeciderParams: Seq[EnableContentFeaturesHydrationParam.type] = Seq(
EnableContentFeaturesHydrationParam
)
val booleanFeatureSwitchParams: Seq[FSParam[Boolean]] = Seq(
EnableExcludeSourceTweetIdsQueryParam,
EnableTokensInContentFeaturesHydrationParam,
EnableReplyRootTweetHydrationParam,
EnableTweetTextInContentFeaturesHydrationParam,
EnableConversationControlInContentFeaturesHydrationParam,
EnableTweetMediaHydrationParam,
EnableEarlybirdReturnAllResultsParam,
EnableEarlybirdRealtimeCgMigrationParam,
RecycledMaxFollowedUsersEnableAntiDilutionParam
)
val boundedIntFeatureSwitchParams: Seq[FSBoundedParam[Int]] = Seq(
MaxFollowedUsersParam,
RelevanceOptionsMaxHitsToProcessParam
)
}
class InNetworkTweetProduction(deciderGateBuilder: DeciderGateBuilder) {
val configHelper: ConfigHelper =
new ConfigHelper(InNetworkTweetProduction.deciderByParam, deciderGateBuilder)
val doubleDeciderOverrides: Seq[OptionalOverride[Double]] =
configHelper.createDeciderBasedOverrides(InNetworkTweetProduction.doubleParams)
val booleanDeciderOverrides: Seq[OptionalOverride[Boolean]] =
configHelper.createDeciderBasedBooleanOverrides(InNetworkTweetProduction.booleanDeciderParams)
val boundedIntFeatureSwitchOverrides: Seq[OptionalOverride[Int]] =
FeatureSwitchOverrideUtil.getBoundedIntFSOverrides(
InNetworkTweetProduction.boundedIntFeatureSwitchParams: _*)
val booleanFeatureSwitchOverrides: Seq[OptionalOverride[Boolean]] =
FeatureSwitchOverrideUtil.getBooleanFSOverrides(
InNetworkTweetProduction.booleanFeatureSwitchParams: _*)
val config: BaseConfig = new BaseConfigBuilder()
.set(
booleanDeciderOverrides: _*
)
.set(
doubleDeciderOverrides: _*
)
.set(
boundedIntFeatureSwitchOverrides: _*
)
.set(
booleanFeatureSwitchOverrides: _*
)
.build(InNetworkTweetProduction.getClass.getSimpleName)
}

11
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring/BUILD Normal file
View File

@ -0,0 +1,11 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
"servo/decider",
"servo/util/src/main/scala",
],
)

13
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring/MonitoringParams.scala Normal file
View File

@ -0,0 +1,13 @@
package com.twitter.timelineranker.parameters.monitoring
import com.twitter.timelines.configapi.FSParam
object MonitoringParams {
object DebugAuthorsAllowListParam
extends FSParam[Seq[Long]](
name = "monitoring_debug_authors_allow_list",
default = Seq.empty[Long]
)
}

14
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring/MonitoringProduction.scala Normal file
View File

@ -0,0 +1,14 @@
package com.twitter.timelineranker.parameters.monitoring
import com.twitter.timelines.configapi.BaseConfigBuilder
import com.twitter.timelineranker.parameters.monitoring.MonitoringParams.DebugAuthorsAllowListParam
import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil
object MonitoringProduction {
private val longSeqOverrides =
FeatureSwitchOverrideUtil.getLongSeqFSOverrides(DebugAuthorsAllowListParam)
val config = BaseConfigBuilder()
.set(longSeqOverrides: _*)
.build()
}

18
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/BUILD Normal file
View File

@ -0,0 +1,18 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
"configapi/configapi-decider/src/main/scala/com/twitter/timelines/configapi/decider",
"servo/decider",
"servo/util/src/main/scala",
"timelineranker/common/src/main/scala/com/twitter/timelineranker/model",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/decider",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util",
"timelines/src/main/scala/com/twitter/timelines/experiment",
"util/util-logging",
"util/util-stats",
],
)

231
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/RecapParams.scala Normal file
View File

@ -0,0 +1,231 @@
package com.twitter.timelineranker.parameters.recap
import com.twitter.timelines.configapi.decider._
import com.twitter.timelines.configapi.FSBoundedParam
import com.twitter.timelines.configapi.FSParam
import com.twitter.timelines.configapi.Param
import com.twitter.timelines.util.bounds.BoundsWithDefault
object RecapParams {
val MaxFollowedUsers: BoundsWithDefault[Int] = BoundsWithDefault[Int](1, 3000, 1000)
val MaxCountMultiplier: BoundsWithDefault[Double] = BoundsWithDefault[Double](0.1, 2.0, 2.0)
val MaxRealGraphAndFollowedUsers: BoundsWithDefault[Int] = BoundsWithDefault[Int](0, 2000, 1000)
val ProbabilityRandomTweet: BoundsWithDefault[Double] = BoundsWithDefault[Double](0.0, 1.0, 0.0)
/**
* Controls limit on the number of followed users fetched from SGS.
*
* The specific default value below is for blender-timelines parity.
*/
object MaxFollowedUsersParam
extends FSBoundedParam[Int](
name = "recap_max_followed_users",
default = MaxFollowedUsers.default,
min = MaxFollowedUsers.bounds.minInclusive,
max = MaxFollowedUsers.bounds.maxInclusive
)
/**
* Controls limit on the number of hits for Earlybird.
* We added it solely for backward compatibility, to align with recycled.
* RecapSource is deprecated, but, this param is used by RecapAuthor source
*/
object RelevanceOptionsMaxHitsToProcessParam
extends FSBoundedParam[Int](
name = "recap_relevance_options_max_hits_to_process",
default = 500,
min = 100,
max = 20000
)
/**
* Enables fetching author seedset from real graph users. Only used if user follows >= 1000.
* If true, expands author seedset with real graph users and recent followed users.
* Otherwise, user seedset only includes followed users.
*/
object EnableRealGraphUsersParam extends Param(false)
/**
* Only used if EnableRealGraphUsersParam is true and OnlyRealGraphUsersParam is false.
* Maximum number of real graph users and recent followed users when mixing recent/real-graph users.
*/
object MaxRealGraphAndFollowedUsersParam
extends Param(MaxRealGraphAndFollowedUsers.default)
with DeciderValueConverter[Int] {
override def convert: IntConverter[Int] =
OutputBoundIntConverter(MaxRealGraphAndFollowedUsers.bounds)
}
/**
* FS-controlled param to override the MaxRealGraphAndFollowedUsersParam decider value for experiments
*/
object MaxRealGraphAndFollowedUsersFSOverrideParam
extends FSBoundedParam[Option[Int]](
name = "max_real_graph_and_followers_users_fs_override_param",
default = None,
min = Some(100),
max = Some(10000)
)
/**
* Experimental params for leveling the playing field between user folowees received from
* real-graph and follow-graph stores.
* Author relevance scores returned by real-graph are currently being used for light-ranking
* in-network tweet candidates.
* Follow-graph store returns the most recent followees without any relevance scores
* We are trying to impute the missing scores by using aggregated statistics (min, avg, p50, etc.)
* of real-graph scores.
*/
object ImputeRealGraphAuthorWeightsParam
extends FSParam(name = "impute_real_graph_author_weights", default = false)
object ImputeRealGraphAuthorWeightsPercentileParam
extends FSBoundedParam[Int](
name = "impute_real_graph_author_weights_percentile",
default = 50,
min = 0,
max = 99)
/**
* Enable running the new pipeline for recap author source
*/
object EnableNewRecapAuthorPipeline extends Param(false)
/**
* Fallback value for maximum number of search results, if not specified by query.maxCount
*/
object DefaultMaxTweetCount extends Param(200)
/**
* We multiply maxCount (caller supplied value) by this multiplier and fetch those many
* candidates from search so that we are left with sufficient number of candidates after
* hydration and filtering.
*/
object MaxCountMultiplierParam
extends Param(MaxCountMultiplier.default)
with DeciderValueConverter[Double] {
override def convert: IntConverter[Double] =
OutputBoundIntConverter[Double](divideDeciderBy100 _, MaxCountMultiplier.bounds)
}
/**
* Enables return all results from search index.
*/
object EnableReturnAllResultsParam
extends FSParam(name = "recap_enable_return_all_results", default = false)
/**
* Includes one or multiple random tweets in the response.
*/
object IncludeRandomTweetParam
extends FSParam(name = "recap_include_random_tweet", default = false)
/**
* One single random tweet (true) or tag tweet as random with given probability (false).
*/
object IncludeSingleRandomTweetParam
extends FSParam(name = "recap_include_random_tweet_single", default = true)
/**
* Probability to tag a tweet as random (will not be ranked).
*/
object ProbabilityRandomTweetParam
extends FSBoundedParam(
name = "recap_include_random_tweet_probability",
default = ProbabilityRandomTweet.default,
min = ProbabilityRandomTweet.bounds.minInclusive,
max = ProbabilityRandomTweet.bounds.maxInclusive)
/**
* Enable extra sorting by score for search results.
*/
object EnableExtraSortingInSearchResultParam extends Param(true)
/**
* Enables semantic core, penguin, and tweetypie content features in recap source.
*/
object EnableContentFeaturesHydrationParam extends Param(true)
/**
* additionally enables tokens when hydrating content features.
*/
object EnableTokensInContentFeaturesHydrationParam
extends FSParam(
name = "recap_enable_tokens_in_content_features_hydration",
default = false
)
/**
* additionally enables tweet text when hydrating content features.
* This only works if EnableContentFeaturesHydrationParam is set to true
*/
object EnableTweetTextInContentFeaturesHydrationParam
extends FSParam(
name = "recap_enable_tweet_text_in_content_features_hydration",
default = false
)
/**
* Enables hydrating in-network inReplyToTweet features
*/
object EnableInNetworkInReplyToTweetFeaturesHydrationParam
extends FSParam(
name = "recap_enable_in_network_in_reply_to_tweet_features_hydration",
default = false
)
/**
* Enables hydrating root tweet of in-network replies and extended replies
*/
object EnableReplyRootTweetHydrationParam
extends FSParam(
name = "recap_enable_reply_root_tweet_hydration",
default = false
)
/**
* Enable setting tweetTypes in search queries with TweetKindOption in RecapQuery
*/
object EnableSettingTweetTypesWithTweetKindOption
extends FSParam(
name = "recap_enable_setting_tweet_types_with_tweet_kind_option",
default = false
)
/**
* Enable relevance search, otherwise recency search from earlybird.
*/
object EnableRelevanceSearchParam
extends FSParam(
name = "recap_enable_relevance_search",
default = true
)
object EnableExpandedExtendedRepliesFilterParam
extends FSParam(
name = "recap_enable_expanded_extended_replies_filter",
default = false
)
/**
* additionally enables conversationControl when hydrating content features.
* This only works if EnableContentFeaturesHydrationParam is set to true
*/
object EnableConversationControlInContentFeaturesHydrationParam
extends FSParam(
name = "conversation_control_in_content_features_hydration_recap_enable",
default = false
)
object EnableTweetMediaHydrationParam
extends FSParam(
name = "tweet_media_hydration_recap_enable",
default = false
)
object EnableExcludeSourceTweetIdsQueryParam
extends FSParam[Boolean](
name = "recap_exclude_source_tweet_ids_query_enable",
default = false
)
}

115
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/RecapProduction.scala Normal file
View File

@ -0,0 +1,115 @@
package com.twitter.timelineranker.parameters.recap
import com.twitter.finagle.stats.StatsReceiver
import com.twitter.servo.decider.DeciderGateBuilder
import com.twitter.servo.decider.DeciderKeyName
import com.twitter.timelineranker.decider.DeciderKey
import com.twitter.timelineranker.parameters.recap.RecapParams._
import com.twitter.timelineranker.parameters.util.ConfigHelper
import com.twitter.timelines.configapi._
import com.twitter.servo.decider.DeciderKeyEnum
object RecapProduction {
val deciderByParam: Map[Param[_], DeciderKeyEnum#Value] = Map[Param[_], DeciderKeyName](
EnableRealGraphUsersParam -> DeciderKey.EnableRealGraphUsers,
MaxRealGraphAndFollowedUsersParam -> DeciderKey.MaxRealGraphAndFollowedUsers,
EnableContentFeaturesHydrationParam -> DeciderKey.RecapEnableContentFeaturesHydration,
MaxCountMultiplierParam -> DeciderKey.RecapMaxCountMultiplier,
EnableNewRecapAuthorPipeline -> DeciderKey.RecapAuthorEnableNewPipeline,
RecapParams.EnableExtraSortingInSearchResultParam -> DeciderKey.RecapEnableExtraSortingInResults
)
val intParams: Seq[MaxRealGraphAndFollowedUsersParam.type] = Seq(
MaxRealGraphAndFollowedUsersParam
)
val doubleParams: Seq[MaxCountMultiplierParam.type] = Seq(
MaxCountMultiplierParam
)
val boundedDoubleFeatureSwitchParams: Seq[FSBoundedParam[Double]] = Seq(
RecapParams.ProbabilityRandomTweetParam
)
val booleanParams: Seq[Param[Boolean]] = Seq(
EnableRealGraphUsersParam,
EnableContentFeaturesHydrationParam,
EnableNewRecapAuthorPipeline,
RecapParams.EnableExtraSortingInSearchResultParam
)
val booleanFeatureSwitchParams: Seq[FSParam[Boolean]] = Seq(
RecapParams.EnableReturnAllResultsParam,
RecapParams.IncludeRandomTweetParam,
RecapParams.IncludeSingleRandomTweetParam,
RecapParams.EnableInNetworkInReplyToTweetFeaturesHydrationParam,
RecapParams.EnableReplyRootTweetHydrationParam,
RecapParams.EnableSettingTweetTypesWithTweetKindOption,
RecapParams.EnableRelevanceSearchParam,
EnableTokensInContentFeaturesHydrationParam,
EnableTweetTextInContentFeaturesHydrationParam,
EnableExpandedExtendedRepliesFilterParam,
EnableConversationControlInContentFeaturesHydrationParam,
EnableTweetMediaHydrationParam,
ImputeRealGraphAuthorWeightsParam,
EnableExcludeSourceTweetIdsQueryParam
)
val boundedIntFeatureSwitchParams: Seq[FSBoundedParam[Int]] = Seq(
RecapParams.MaxFollowedUsersParam,
ImputeRealGraphAuthorWeightsPercentileParam,
RecapParams.RelevanceOptionsMaxHitsToProcessParam
)
}
class RecapProduction(deciderGateBuilder: DeciderGateBuilder, statsReceiver: StatsReceiver) {
val configHelper: ConfigHelper =
new ConfigHelper(RecapProduction.deciderByParam, deciderGateBuilder)
val intOverrides: Seq[OptionalOverride[Int]] =
configHelper.createDeciderBasedOverrides(RecapProduction.intParams)
val optionalBoundedIntFeatureSwitchOverrides: Seq[OptionalOverride[Option[Int]]] =
FeatureSwitchOverrideUtil.getBoundedOptionalIntOverrides(
(
MaxRealGraphAndFollowedUsersFSOverrideParam,
"max_real_graph_and_followers_users_fs_override_defined",
"max_real_graph_and_followers_users_fs_override_value"
)
)
val doubleOverrides: Seq[OptionalOverride[Double]] =
configHelper.createDeciderBasedOverrides(RecapProduction.doubleParams)
val booleanOverrides: Seq[OptionalOverride[Boolean]] =
configHelper.createDeciderBasedBooleanOverrides(RecapProduction.booleanParams)
val booleanFeatureSwitchOverrides: Seq[OptionalOverride[Boolean]] =
FeatureSwitchOverrideUtil.getBooleanFSOverrides(RecapProduction.booleanFeatureSwitchParams: _*)
val boundedDoubleFeatureSwitchOverrides: Seq[OptionalOverride[Double]] =
FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(
RecapProduction.boundedDoubleFeatureSwitchParams: _*)
val boundedIntFeatureSwitchOverrides: Seq[OptionalOverride[Int]] =
FeatureSwitchOverrideUtil.getBoundedIntFSOverrides(
RecapProduction.boundedIntFeatureSwitchParams: _*)
val config: BaseConfig = new BaseConfigBuilder()
.set(
intOverrides: _*
)
.set(
booleanOverrides: _*
)
.set(
doubleOverrides: _*
)
.set(
booleanFeatureSwitchOverrides: _*
)
.set(
boundedIntFeatureSwitchOverrides: _*
)
.set(
optionalBoundedIntFeatureSwitchOverrides: _*
)
.set(
boundedDoubleFeatureSwitchOverrides: _*
)
.build(RecapProduction.getClass.getSimpleName)
}

79
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap/RecapQueryContext.scala Normal file
View File

@ -0,0 +1,79 @@
package com.twitter.timelineranker.parameters.recap
import com.twitter.timelineranker.model.RecapQuery
import com.twitter.timelines.util.bounds.BoundsWithDefault
object RecapQueryContext {
val MaxFollowedUsers: BoundsWithDefault[Int] = BoundsWithDefault[Int](1, 3000, 1000)
val MaxCountMultiplier: BoundsWithDefault[Double] = BoundsWithDefault[Double](0.1, 2.0, 2.0)
val MaxRealGraphAndFollowedUsers: BoundsWithDefault[Int] = BoundsWithDefault[Int](0, 2000, 1000)
def getDefaultContext(query: RecapQuery): RecapQueryContext = {
new RecapQueryContextImpl(
query,
getEnableHydrationUsingTweetyPie = () => false,
getMaxFollowedUsers = () => MaxFollowedUsers.default,
getMaxCountMultiplier = () => MaxCountMultiplier.default,
getEnableRealGraphUsers = () => false,
getOnlyRealGraphUsers = () => false,
getMaxRealGraphAndFollowedUsers = () => MaxRealGraphAndFollowedUsers.default,
getEnableTextFeatureHydration = () => false
)
}
}
// Note that methods that return parameter value always use () to indicate that
// side effects may be involved in their invocation.
trait RecapQueryContext {
def query: RecapQuery
// If true, tweet hydration are performed by calling TweetyPie.
// Otherwise, tweets are partially hydrated based on information in ThriftSearchResult.
def enableHydrationUsingTweetyPie(): Boolean
// Maximum number of followed user accounts to use when fetching recap tweets.
def maxFollowedUsers(): Int
// We multiply maxCount (caller supplied value) by this multiplier and fetch those many
// candidates from search so that we are left with sufficient number of candidates after
// hydration and filtering.
def maxCountMultiplier(): Double
// Only used if user follows >= 1000.
// If true, fetches recap/recycled tweets using author seedset mixing with real graph users and followed users.
// Otherwise, fetches recap/recycled tweets only using followed users
def enableRealGraphUsers(): Boolean
// Only used if enableRealGraphUsers is true.
// If true, user seedset only contains real graph users.
// Otherwise, user seedset contains real graph users and recent followed users.
def onlyRealGraphUsers(): Boolean
// Only used if enableRealGraphUsers is true and onlyRealGraphUsers is false.
// Maximum number of real graph users and recent followed users when mixing recent/real-graph users.
def maxRealGraphAndFollowedUsers(): Int
// If true, text features are hydrated for prediction.
// Otherwise those feature values are not set at all.
def enableTextFeatureHydration(): Boolean
}
class RecapQueryContextImpl(
override val query: RecapQuery,
getEnableHydrationUsingTweetyPie: () => Boolean,
getMaxFollowedUsers: () => Int,
getMaxCountMultiplier: () => Double,
getEnableRealGraphUsers: () => Boolean,
getOnlyRealGraphUsers: () => Boolean,
getMaxRealGraphAndFollowedUsers: () => Int,
getEnableTextFeatureHydration: () => Boolean)
extends RecapQueryContext {
override def enableHydrationUsingTweetyPie(): Boolean = { getEnableHydrationUsingTweetyPie() }
override def maxFollowedUsers(): Int = { getMaxFollowedUsers() }
override def maxCountMultiplier(): Double = { getMaxCountMultiplier() }
override def enableRealGraphUsers(): Boolean = { getEnableRealGraphUsers() }
override def onlyRealGraphUsers(): Boolean = { getOnlyRealGraphUsers() }
override def maxRealGraphAndFollowedUsers(): Int = { getMaxRealGraphAndFollowedUsers() }
override def enableTextFeatureHydration(): Boolean = { getEnableTextFeatureHydration() }
}

12
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author/BUILD Normal file
View File

@ -0,0 +1,12 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
"servo/decider",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/decider",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util",
],
)

53
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author/RecapAuthorParams.scala Normal file
View File

@ -0,0 +1,53 @@
package com.twitter.timelineranker.parameters.recap_author
import com.twitter.timelines.configapi.FSParam
import com.twitter.timelines.configapi.Param
object RecapAuthorParams {
/**
* Enables semantic core, penguin, and tweetypie content features in recap author source.
*/
object EnableContentFeaturesHydrationParam extends Param(false)
/**
* additionally enables tokens when hydrating content features.
*/
object EnableTokensInContentFeaturesHydrationParam
extends FSParam(
name = "recap_author_enable_tokens_in_content_features_hydration",
default = false
)
/**
* additionally enables tweet text when hydrating content features.
* This only works if EnableContentFeaturesHydrationParam is set to true
*/
object EnableTweetTextInContentFeaturesHydrationParam
extends FSParam(
name = "recap_author_enable_tweet_text_in_content_features_hydration",
default = false
)
object EnableEarlybirdRealtimeCgMigrationParam
extends FSParam(
name = "recap_author_enable_earlybird_realtime_cg_migration",
default = false
)
/**
* additionally enables conversationControl when hydrating content features.
* This only works if EnableContentFeaturesHydrationParam is set to true
*/
object EnableConversationControlInContentFeaturesHydrationParam
extends FSParam(
name = "conversation_control_in_content_features_hydration_recap_author_enable",
default = false
)
object EnableTweetMediaHydrationParam
extends FSParam(
name = "tweet_media_hydration_recap_author_enable",
default = false
)
}

46
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author/RecapAuthorProduction.scala Normal file
View File

@ -0,0 +1,46 @@
package com.twitter.timelineranker.parameters.recap_author
import com.twitter.servo.decider.DeciderGateBuilder
import com.twitter.servo.decider.DeciderKeyName
import com.twitter.timelineranker.decider.DeciderKey
import com.twitter.timelineranker.parameters.recap_author.RecapAuthorParams._
import com.twitter.timelineranker.parameters.util.ConfigHelper
import com.twitter.timelines.configapi._
object RecapAuthorProduction {
val deciderByParam: Map[Param[_], DeciderKeyName] = Map[Param[_], DeciderKeyName](
EnableContentFeaturesHydrationParam -> DeciderKey.RecapAuthorEnableContentFeaturesHydration
)
val booleanParams: Seq[EnableContentFeaturesHydrationParam.type] = Seq(
EnableContentFeaturesHydrationParam
)
val booleanFeatureSwitchParams: Seq[FSParam[Boolean]] = Seq(
EnableTokensInContentFeaturesHydrationParam,
EnableTweetTextInContentFeaturesHydrationParam,
EnableConversationControlInContentFeaturesHydrationParam,
EnableTweetMediaHydrationParam,
EnableEarlybirdRealtimeCgMigrationParam
)
}
class RecapAuthorProduction(deciderGateBuilder: DeciderGateBuilder) {
val configHelper: ConfigHelper =
new ConfigHelper(RecapAuthorProduction.deciderByParam, deciderGateBuilder)
val booleanOverrides: Seq[OptionalOverride[Boolean]] =
configHelper.createDeciderBasedBooleanOverrides(RecapAuthorProduction.booleanParams)
val booleanFeatureSwitchOverrides: Seq[OptionalOverride[Boolean]] =
FeatureSwitchOverrideUtil.getBooleanFSOverrides(
RecapAuthorProduction.booleanFeatureSwitchParams: _*
)
val config: BaseConfig = new BaseConfigBuilder()
.set(
booleanOverrides: _*
).set(
booleanFeatureSwitchOverrides: _*
)
.build(RecapAuthorProduction.getClass.getSimpleName)
}

12
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration/BUILD Normal file
View File

@ -0,0 +1,12 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
"servo/decider",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/decider",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util",
],
)

48
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration/RecapHydrationParams.scala Normal file
View File

@ -0,0 +1,48 @@
package com.twitter.timelineranker.parameters.recap_hydration
import com.twitter.timelines.configapi.FSParam
import com.twitter.timelines.configapi.Param
object RecapHydrationParams {
/**
* Enables semantic core, penguin, and tweetypie content features in recap hydration source.
*/
object EnableContentFeaturesHydrationParam extends Param(false)
/**
* additionally enables tokens when hydrating content features.
*/
object EnableTokensInContentFeaturesHydrationParam
extends FSParam(
name = "recap_hydration_enable_tokens_in_content_features_hydration",
default = false
)
/**
* additionally enables tweet text when hydrating content features.
* This only works if EnableContentFeaturesHydrationParam is set to true
*/
object EnableTweetTextInContentFeaturesHydrationParam
extends FSParam(
name = "recap_hydration_enable_tweet_text_in_content_features_hydration",
default = false
)
/**
* additionally enables conversationControl when hydrating content features.
* This only works if EnableContentFeaturesHydrationParam is set to true
*/
object EnableConversationControlInContentFeaturesHydrationParam
extends FSParam(
name = "conversation_control_in_content_features_hydration_recap_hydration_enable",
default = false
)
object EnableTweetMediaHydrationParam
extends FSParam(
name = "tweet_media_hydration_recap_hydration_enable",
default = false
)
}

45
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration/RecapHydrationProduction.scala Normal file
View File

@ -0,0 +1,45 @@
package com.twitter.timelineranker.parameters.recap_hydration
import com.twitter.servo.decider.DeciderGateBuilder
import com.twitter.servo.decider.DeciderKeyName
import com.twitter.timelineranker.decider.DeciderKey
import com.twitter.timelineranker.parameters.recap_hydration.RecapHydrationParams._
import com.twitter.timelineranker.parameters.util.ConfigHelper
import com.twitter.timelines.configapi._
object RecapHydrationProduction {
val deciderByParam: Map[Param[_], DeciderKeyName] = Map[Param[_], DeciderKeyName](
EnableContentFeaturesHydrationParam -> DeciderKey.RecapHydrationEnableContentFeaturesHydration
)
val booleanParams: Seq[EnableContentFeaturesHydrationParam.type] = Seq(
EnableContentFeaturesHydrationParam
)
val booleanFeatureSwitchParams: Seq[FSParam[Boolean]] = Seq(
EnableTokensInContentFeaturesHydrationParam,
EnableTweetTextInContentFeaturesHydrationParam,
EnableConversationControlInContentFeaturesHydrationParam,
EnableTweetMediaHydrationParam
)
}
class RecapHydrationProduction(deciderGateBuilder: DeciderGateBuilder) {
val configHelper: ConfigHelper =
new ConfigHelper(RecapHydrationProduction.deciderByParam, deciderGateBuilder)
val booleanOverrides: Seq[OptionalOverride[Boolean]] =
configHelper.createDeciderBasedBooleanOverrides(RecapHydrationProduction.booleanParams)
val booleanFeatureSwitchOverrides: Seq[OptionalOverride[Boolean]] =
FeatureSwitchOverrideUtil.getBooleanFSOverrides(
RecapHydrationProduction.booleanFeatureSwitchParams: _*
)
val config: BaseConfig = new BaseConfigBuilder()
.set(
booleanOverrides: _*
).set(
booleanFeatureSwitchOverrides: _*
)
.build(RecapHydrationProduction.getClass.getSimpleName)
}

18
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/BUILD Normal file
View File

@ -0,0 +1,18 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
tags = ["bazel-compatible"],
dependencies = [
"configapi/configapi-core",
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
"servo/decider",
"timelineranker/common/src/main/scala/com/twitter/timelineranker/model",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/config",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/decider",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util",
"timelines/src/main/scala/com/twitter/timelines/config",
"timelines/src/main/scala/com/twitter/timelines/decider",
"timelines/src/main/scala/com/twitter/timelines/util/bounds",
"util/util-stats/src/main/scala",
],
)

45
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronParams.scala Normal file
View File

@ -0,0 +1,45 @@
package com.twitter.timelineranker.parameters.revchron
import com.twitter.timelines.configapi.FSBoundedParam
import com.twitter.timelines.configapi.FSParam
object ReverseChronParams {
import ReverseChronTimelineQueryContext._
/**
* Controls limit on the number of followed users fetched from SGS when materializing home timelines.
*/
object MaxFollowedUsersParam
extends FSBoundedParam(
"reverse_chron_max_followed_users",
default = MaxFollowedUsers.default,
min = MaxFollowedUsers.bounds.minInclusive,
max = MaxFollowedUsers.bounds.maxInclusive
)
object ReturnEmptyWhenOverMaxFollowsParam
extends FSParam(
name = "reverse_chron_return_empty_when_over_max_follows",
default = true
)
/**
* When true, search requests for the reverse chron timeline will include an additional operator
* so that search will not return tweets that are directed at non-followed users.
*/
object DirectedAtNarrowcastingViaSearchParam
extends FSParam(
name = "reverse_chron_directed_at_narrowcasting_via_search",
default = false
)
/**
* When true, search requests for the reverse chron timeline will request additional metadata
* from search and use this metadata for post filtering.
*/
object PostFilteringBasedOnSearchMetadataEnabledParam
extends FSParam(
name = "reverse_chron_post_filtering_based_on_search_metadata_enabled",
default = true
)
}

28
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronProduction.scala Normal file
View File

@ -0,0 +1,28 @@
package com.twitter.timelineranker.parameters.revchron
import com.twitter.servo.decider.DeciderGateBuilder
import com.twitter.timelines.configapi._
object ReverseChronProduction {
val intFeatureSwitchParams = Seq(ReverseChronParams.MaxFollowedUsersParam)
val booleanFeatureSwitchParams = Seq(
ReverseChronParams.ReturnEmptyWhenOverMaxFollowsParam,
ReverseChronParams.DirectedAtNarrowcastingViaSearchParam,
ReverseChronParams.PostFilteringBasedOnSearchMetadataEnabledParam
)
}
class ReverseChronProduction(deciderGateBuilder: DeciderGateBuilder) {
val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides(
ReverseChronProduction.intFeatureSwitchParams: _*
)
val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides(
ReverseChronProduction.booleanFeatureSwitchParams: _*
)
val config: BaseConfig = new BaseConfigBuilder()
.set(intOverrides: _*)
.set(booleanOverrides: _*)
.build(ReverseChronProduction.getClass.getSimpleName)
}

114
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronTimelineQueryContext.scala Normal file
View File

@ -0,0 +1,114 @@
package com.twitter.timelineranker.parameters.revchron
import com.twitter.timelineranker.model.ReverseChronTimelineQuery
import com.twitter.timelines.util.bounds.BoundsWithDefault
import com.twitter.timelineservice.model.core.TimelineKind
import com.twitter.timelineservice.model.core.TimelineLimits
object ReverseChronTimelineQueryContext {
val MaxCountLimit: Int = TimelineLimits.default.lengthLimit(TimelineKind.home)
val MaxCount: BoundsWithDefault[Int] = BoundsWithDefault[Int](0, MaxCountLimit, MaxCountLimit)
val MaxCountMultiplier: BoundsWithDefault[Double] = BoundsWithDefault[Double](0.5, 2.0, 1.0)
val MaxFollowedUsers: BoundsWithDefault[Int] = BoundsWithDefault[Int](1, 15000, 5000)
val TweetsFilteringLossageThresholdPercent: BoundsWithDefault[Int] =
BoundsWithDefault[Int](10, 100, 20)
val TweetsFilteringLossageLimitPercent: BoundsWithDefault[Int] =
BoundsWithDefault[Int](40, 65, 60)
def getDefaultContext(query: ReverseChronTimelineQuery): ReverseChronTimelineQueryContext = {
new ReverseChronTimelineQueryContextImpl(
query,
getMaxCount = () => MaxCount.default,
getMaxCountMultiplier = () => MaxCountMultiplier.default,
getMaxFollowedUsers = () => MaxFollowedUsers.default,
getReturnEmptyWhenOverMaxFollows = () => true,
getDirectedAtNarrowastingViaSearch = () => false,
getPostFilteringBasedOnSearchMetadataEnabled = () => true,
getBackfillFilteredEntries = () => false,
getTweetsFilteringLossageThresholdPercent = () =>
TweetsFilteringLossageThresholdPercent.default,
getTweetsFilteringLossageLimitPercent = () => TweetsFilteringLossageLimitPercent.default
)
}
}
// Note that methods that return parameter value always use () to indicate that
// side effects may be involved in their invocation.
// for example, A likely side effect is to cause experiment impression.
trait ReverseChronTimelineQueryContext {
def query: ReverseChronTimelineQuery
// Maximum number of tweets to be returned to caller.
def maxCount(): Int
// Multiplier applied to the number of tweets fetched from search expressed as percentage.
// It can be used to fetch more than the number tweets requested by a caller (to improve similarity)
// or to fetch less than requested to reduce load.
def maxCountMultiplier(): Double
// Maximum number of followed user accounts to use when materializing home timelines.
def maxFollowedUsers(): Int
// When true, if the user follows more than maxFollowedUsers, return an empty timeline.
def returnEmptyWhenOverMaxFollows(): Boolean
// When true, appends an operator for directed-at narrowcasting to the home materialization
// search request
def directedAtNarrowcastingViaSearch(): Boolean
// When true, requests additional metadata from search and use this metadata for post filtering.
def postFilteringBasedOnSearchMetadataEnabled(): Boolean
// Controls whether to back-fill timeline entries that get filtered out by TweetsPostFilter
// during home timeline materialization.
def backfillFilteredEntries(): Boolean
// If back-filling filtered entries is enabled and if number of tweets that get filtered out
// exceed this percentage then we will issue a second call to get more tweets.
def tweetsFilteringLossageThresholdPercent(): Int
// We need to ensure that the number of tweets requested by the second call
// are not unbounded (for example, if everything is filtered out in the first call)
// therefore we adjust the actual filtered out percentage to be no greater than
// the value below.
def tweetsFilteringLossageLimitPercent(): Int
// We need to indicate to search if we should use the archive cluster
// this option will come from ReverseChronTimelineQueryOptions and
// will be `true` by default if the options are not present.
def getTweetsFromArchiveIndex(): Boolean =
query.options.map(_.getTweetsFromArchiveIndex).getOrElse(true)
}
class ReverseChronTimelineQueryContextImpl(
override val query: ReverseChronTimelineQuery,
getMaxCount: () => Int,
getMaxCountMultiplier: () => Double,
getMaxFollowedUsers: () => Int,
getReturnEmptyWhenOverMaxFollows: () => Boolean,
getDirectedAtNarrowastingViaSearch: () => Boolean,
getPostFilteringBasedOnSearchMetadataEnabled: () => Boolean,
getBackfillFilteredEntries: () => Boolean,
getTweetsFilteringLossageThresholdPercent: () => Int,
getTweetsFilteringLossageLimitPercent: () => Int)
extends ReverseChronTimelineQueryContext {
override def maxCount(): Int = { getMaxCount() }
override def maxCountMultiplier(): Double = { getMaxCountMultiplier() }
override def maxFollowedUsers(): Int = { getMaxFollowedUsers() }
override def backfillFilteredEntries(): Boolean = { getBackfillFilteredEntries() }
override def tweetsFilteringLossageThresholdPercent(): Int = {
getTweetsFilteringLossageThresholdPercent()
}
override def tweetsFilteringLossageLimitPercent(): Int = {
getTweetsFilteringLossageLimitPercent()
}
override def returnEmptyWhenOverMaxFollows(): Boolean = {
getReturnEmptyWhenOverMaxFollows()
}
override def directedAtNarrowcastingViaSearch(): Boolean = {
getDirectedAtNarrowastingViaSearch()
}
override def postFilteringBasedOnSearchMetadataEnabled(): Boolean = {
getPostFilteringBasedOnSearchMetadataEnabled()
}
}

72
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron/ReverseChronTimelineQueryContextBuilder.scala Normal file
View File

@ -0,0 +1,72 @@
package com.twitter.timelineranker.parameters.revchron
import com.twitter.timelineranker.config.RuntimeConfiguration
import com.twitter.timelineranker.decider.DeciderKey
import com.twitter.timelineranker.model._
import com.twitter.timelineranker.parameters.util.RequestContextBuilder
import com.twitter.timelines.configapi.Config
import com.twitter.timelines.decider.FeatureValue
import com.twitter.util.Future
object ReverseChronTimelineQueryContextBuilder {
val MaxCountLimitKey: Seq[String] = Seq("search_request_max_count_limit")
}
class ReverseChronTimelineQueryContextBuilder(
config: Config,
runtimeConfig: RuntimeConfiguration,
requestContextBuilder: RequestContextBuilder) {
import ReverseChronTimelineQueryContext._
import ReverseChronTimelineQueryContextBuilder._
private val maxCountMultiplier = FeatureValue(
runtimeConfig.deciderGateBuilder,
DeciderKey.MultiplierOfMaterializationTweetsFetched,
MaxCountMultiplier,
value => (value / 100.0)
)
private val backfillFilteredEntriesGate =
runtimeConfig.deciderGateBuilder.linearGate(DeciderKey.BackfillFilteredEntries)
private val tweetsFilteringLossageThresholdPercent = FeatureValue(
runtimeConfig.deciderGateBuilder,
DeciderKey.TweetsFilteringLossageThreshold,
TweetsFilteringLossageThresholdPercent,
value => (value / 100)
)
private val tweetsFilteringLossageLimitPercent = FeatureValue(
runtimeConfig.deciderGateBuilder,
DeciderKey.TweetsFilteringLossageLimit,
TweetsFilteringLossageLimitPercent,
value => (value / 100)
)
private def getMaxCountFromConfigStore(): Int = {
runtimeConfig.configStore.getAsInt(MaxCountLimitKey).getOrElse(MaxCount.default)
}
def apply(query: ReverseChronTimelineQuery): Future[ReverseChronTimelineQueryContext] = {
requestContextBuilder(Some(query.userId), deviceContext = None).map { baseContext =>
val params = config(baseContext, runtimeConfig.statsReceiver)
new ReverseChronTimelineQueryContextImpl(
query,
getMaxCount = () => getMaxCountFromConfigStore(),
getMaxCountMultiplier = () => maxCountMultiplier(),
getMaxFollowedUsers = () => params(ReverseChronParams.MaxFollowedUsersParam),
getReturnEmptyWhenOverMaxFollows =
() => params(ReverseChronParams.ReturnEmptyWhenOverMaxFollowsParam),
getDirectedAtNarrowastingViaSearch =
() => params(ReverseChronParams.DirectedAtNarrowcastingViaSearchParam),
getPostFilteringBasedOnSearchMetadataEnabled =
() => params(ReverseChronParams.PostFilteringBasedOnSearchMetadataEnabledParam),
getBackfillFilteredEntries = () => backfillFilteredEntriesGate(),
getTweetsFilteringLossageThresholdPercent = () => tweetsFilteringLossageThresholdPercent(),
getTweetsFilteringLossageLimitPercent = () => tweetsFilteringLossageLimitPercent()
)
}
}
}

13
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets/BUILD Normal file
View File

@ -0,0 +1,13 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
"servo/decider/src/main/scala",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/decider",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util",
"timelines/src/main/scala/com/twitter/timelines/util/bounds",
],
)

174
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets/UtegLikedByTweetsParams.scala Normal file
View File

@ -0,0 +1,174 @@
package com.twitter.timelineranker.parameters.uteg_liked_by_tweets
import com.twitter.timelines.configapi.FSBoundedParam
import com.twitter.timelines.configapi.FSParam
import com.twitter.timelines.configapi.Param
import com.twitter.timelines.util.bounds.BoundsWithDefault
object UtegLikedByTweetsParams {
val ProbabilityRandomTweet: BoundsWithDefault[Double] = BoundsWithDefault[Double](0.0, 1.0, 0.0)
object DefaultUTEGInNetworkCount extends Param(200)
object DefaultUTEGOutOfNetworkCount extends Param(800)
object DefaultMaxTweetCount extends Param(200)
/**
* Enables semantic core, penguin, and tweetypie content features in uteg liked by tweets source.
*/
object EnableContentFeaturesHydrationParam extends Param(false)
/**
* additionally enables tokens when hydrating content features.
*/
object EnableTokensInContentFeaturesHydrationParam
extends FSParam(
name = "uteg_liked_by_tweets_enable_tokens_in_content_features_hydration",
default = false
)
/**
* additionally enables tweet text when hydrating content features.
* This only works if EnableContentFeaturesHydrationParam is set to true
*/
object EnableTweetTextInContentFeaturesHydrationParam
extends FSParam(
name = "uteg_liked_by_tweets_enable_tweet_text_in_content_features_hydration",
default = false
)
/**
* A multiplier for earlybird score when combining earlybird score and real graph score for ranking.
* Note multiplier for realgraph score := 1.0, and only change earlybird score multiplier.
*/
object EarlybirdScoreMultiplierParam
extends FSBoundedParam(
"uteg_liked_by_tweets_earlybird_score_multiplier_param",
1.0,
0,
20.0
)
object UTEGRecommendationsFilter {
/**
* enable filtering of UTEG recommendations based on social proof type
*/
object EnableParam
extends FSParam(
"uteg_liked_by_tweets_uteg_recommendations_filter_enable",
false
)
/**
* filters out UTEG recommendations that have been tweeted by someone the user follows
*/
object ExcludeTweetParam
extends FSParam(
"uteg_liked_by_tweets_uteg_recommendations_filter_exclude_tweet",
false
)
/**
* filters out UTEG recommendations that have been retweeted by someone the user follows
*/
object ExcludeRetweetParam
extends FSParam(
"uteg_liked_by_tweets_uteg_recommendations_filter_exclude_retweet",
false
)
/**
* filters out UTEG recommendations that have been replied to by someone the user follows
* not filtering out the replies
*/
object ExcludeReplyParam
extends FSParam(
"uteg_liked_by_tweets_uteg_recommendations_filter_exclude_reply",
false
)
/**
* filters out UTEG recommendations that have been quoted by someone the user follows
*/
object ExcludeQuoteTweetParam
extends FSParam(
"uteg_liked_by_tweets_uteg_recommendations_filter_exclude_quote",
false
)
/**
* filters out recommended replies that have been directed at out of network users.
*/
object ExcludeRecommendedRepliesToNonFollowedUsersParam
extends FSParam(
name =
"uteg_liked_by_tweets_uteg_recommendations_filter_exclude_recommended_replies_to_non_followed_users",
default = false
)
}
/**
* Minimum number of favorited-by users required for recommended tweets.
*/
object MinNumFavoritedByUserIdsParam extends Param(1)
/**
* Includes one or multiple random tweets in the response.
*/
object IncludeRandomTweetParam
extends FSParam(name = "uteg_liked_by_tweets_include_random_tweet", default = false)
/**
* One single random tweet (true) or tag tweet as random with given probability (false).
*/
object IncludeSingleRandomTweetParam
extends FSParam(name = "uteg_liked_by_tweets_include_random_tweet_single", default = false)
/**
* Probability to tag a tweet as random (will not be ranked).
*/
object ProbabilityRandomTweetParam
extends FSBoundedParam(
name = "uteg_liked_by_tweets_include_random_tweet_probability",
default = ProbabilityRandomTweet.default,
min = ProbabilityRandomTweet.bounds.minInclusive,
max = ProbabilityRandomTweet.bounds.maxInclusive)
/**
* additionally enables conversationControl when hydrating content features.
* This only works if EnableContentFeaturesHydrationParam is set to true
*/
object EnableConversationControlInContentFeaturesHydrationParam
extends FSParam(
name = "conversation_control_in_content_features_hydration_uteg_liked_by_tweets_enable",
default = false
)
object EnableTweetMediaHydrationParam
extends FSParam(
name = "tweet_media_hydration_uteg_liked_by_tweets_enable",
default = false
)
object NumAdditionalRepliesParam
extends FSBoundedParam(
name = "uteg_liked_by_tweets_num_additional_replies",
default = 0,
min = 0,
max = 1000
)
/**
* Enable relevance search, otherwise recency search from earlybird.
*/
object EnableRelevanceSearchParam
extends FSParam(
name = "uteg_liked_by_tweets_enable_relevance_search",
default = true
)
}

87
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets/UtegLikedByTweetsProduction.scala Normal file
View File

@ -0,0 +1,87 @@
package com.twitter.timelineranker.parameters.uteg_liked_by_tweets
import com.twitter.servo.decider.DeciderGateBuilder
import com.twitter.servo.decider.DeciderKeyName
import com.twitter.timelineranker.decider.DeciderKey
import com.twitter.timelineranker.parameters.uteg_liked_by_tweets.UtegLikedByTweetsParams._
import com.twitter.timelineranker.parameters.util.ConfigHelper
import com.twitter.timelines.configapi.BaseConfig
import com.twitter.timelines.configapi.BaseConfigBuilder
import com.twitter.timelines.configapi.FSBoundedParam
import com.twitter.timelines.configapi.FSParam
import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil
import com.twitter.timelines.configapi.OptionalOverride
import com.twitter.timelines.configapi.Param
object UtegLikedByTweetsProduction {
val deciderByParam: Map[Param[_], DeciderKeyName] = Map[Param[_], DeciderKeyName](
EnableContentFeaturesHydrationParam -> DeciderKey.UtegLikedByTweetsEnableContentFeaturesHydration
)
val booleanDeciderParams: Seq[EnableContentFeaturesHydrationParam.type] = Seq(
EnableContentFeaturesHydrationParam
)
val intParams: Seq[Param[Int]] = Seq(
DefaultUTEGInNetworkCount,
DefaultMaxTweetCount,
DefaultUTEGOutOfNetworkCount,
MinNumFavoritedByUserIdsParam
)
val booleanFeatureSwitchParams: Seq[FSParam[Boolean]] = Seq(
UTEGRecommendationsFilter.EnableParam,
UTEGRecommendationsFilter.ExcludeQuoteTweetParam,
UTEGRecommendationsFilter.ExcludeReplyParam,
UTEGRecommendationsFilter.ExcludeRetweetParam,
UTEGRecommendationsFilter.ExcludeTweetParam,
EnableTokensInContentFeaturesHydrationParam,
EnableConversationControlInContentFeaturesHydrationParam,
UTEGRecommendationsFilter.ExcludeRecommendedRepliesToNonFollowedUsersParam,
EnableTweetTextInContentFeaturesHydrationParam,
EnableTweetMediaHydrationParam,
UtegLikedByTweetsParams.IncludeRandomTweetParam,
UtegLikedByTweetsParams.IncludeSingleRandomTweetParam,
UtegLikedByTweetsParams.EnableRelevanceSearchParam
)
val boundedDoubleFeatureSwitchParams: Seq[FSBoundedParam[Double]] = Seq(
EarlybirdScoreMultiplierParam,
UtegLikedByTweetsParams.ProbabilityRandomTweetParam
)
val boundedIntFeatureSwitchParams: Seq[FSBoundedParam[Int]] = Seq(
UtegLikedByTweetsParams.NumAdditionalRepliesParam
)
}
class UtegLikedByTweetsProduction(deciderGateBuilder: DeciderGateBuilder) {
val configHelper: ConfigHelper =
new ConfigHelper(UtegLikedByTweetsProduction.deciderByParam, deciderGateBuilder)
val booleanDeciderOverrides: Seq[OptionalOverride[Boolean]] =
configHelper.createDeciderBasedBooleanOverrides(
UtegLikedByTweetsProduction.booleanDeciderParams)
val boundedDoubleFeatureSwitchOverrides: Seq[OptionalOverride[Double]] =
FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(
UtegLikedByTweetsProduction.boundedDoubleFeatureSwitchParams: _*)
val booleanFeatureSwitchOverrides: Seq[OptionalOverride[Boolean]] =
FeatureSwitchOverrideUtil.getBooleanFSOverrides(
UtegLikedByTweetsProduction.booleanFeatureSwitchParams: _*)
val boundedIntFeaturesSwitchOverrides: Seq[OptionalOverride[Int]] =
FeatureSwitchOverrideUtil.getBoundedIntFSOverrides(
UtegLikedByTweetsProduction.boundedIntFeatureSwitchParams: _*)
val config: BaseConfig = new BaseConfigBuilder()
.set(
booleanDeciderOverrides: _*
)
.set(
boundedDoubleFeatureSwitchOverrides: _*
)
.set(
booleanFeatureSwitchOverrides: _*
)
.set(
boundedIntFeaturesSwitchOverrides: _*
)
.build(UtegLikedByTweetsProduction.getClass.getSimpleName)
}

24
timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util/BUILD Normal file
View File

@ -0,0 +1,24 @@
scala_library(
sources = ["*.scala"],
compiler_option_sets = ["fatal_warnings"],
strict_deps = True,
tags = ["bazel-compatible"],
dependencies = [
"configapi/configapi-core",
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
"configapi/configapi-decider",
"decider/src/main/scala",
"servo/decider",
"servo/util/src/main/scala",
"timelineranker/common:model",
"timelineranker/server/src/main/scala/com/twitter/timelineranker/config",
"timelines/src/main/scala/com/twitter/timelines/common/model",
"timelines/src/main/scala/com/twitter/timelines/config",
"timelines/src/main/scala/com/twitter/timelines/config/configapi",
"timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils",
"timelines/src/main/scala/com/twitter/timelines/model/candidate",
"timelines/src/main/scala/com/twitter/timelines/util",
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
],
)

Some files were not shown because too many files have changed in this diff Show More