mirror of
https://github.com/twitter/the-algorithm.git
synced 2025-06-10 14:48:16 -05:00
Twitter Recommendation Algorithm
Please note we have force-pushed a new initial commit in order to remove some publicly-available Twitter user information. Note that this process may be required in the future.
This commit is contained in:
twitter-team
17
timelineranker/server/BUILD.bazel
Normal file
17
timelineranker/server/BUILD.bazel
Normal file
@ -0,0 +1,17 @@
|
||||
target(
|
||||
dependencies = [
|
||||
"timelineranker/config",
|
||||
"timelineranker/server/src/main/scala",
|
||||
],
|
||||
)
|
||||
|
||||
jvm_app(
|
||||
name = "bundle",
|
||||
basename = "timelineranker-server-package-dist",
|
||||
binary = "timelineranker/server/src/main/scala:bin",
|
||||
bundles = [bundle(
|
||||
fileset = ["config/**/*"],
|
||||
owning_target = "timelineranker/server/config:files",
|
||||
)],
|
||||
tags = ["bazel-compatible"],
|
||||
)
|
||||
14
timelineranker/server/config/BUILD
Normal file
14
timelineranker/server/config/BUILD
Normal file
@ -0,0 +1,14 @@
|
||||
resources(
|
||||
sources = ["**/*.yml"],
|
||||
)
|
||||
|
||||
# Created for Bazel compatibility.
|
||||
# In Bazel, loose files must be part of a target to be included into a bundle.
|
||||
# See also http://go/bazel-compatibility/bundle_does_not_match_any_files
|
||||
files(
|
||||
name = "files",
|
||||
sources = [
|
||||
"!BUILD",
|
||||
"**/*",
|
||||
],
|
||||
)
|
||||
153
timelineranker/server/config/decider.yml
Normal file
153
timelineranker/server/config/decider.yml
Normal file
@ -0,0 +1,153 @@
|
||||
# Deciders that can be used to control load on TLR or its backends.
|
||||
enable_max_concurrency_limiting:
|
||||
comment: "When enabled, limit maxConcurrency filter. Note: Requires system property maxConcurrency to be set."
|
||||
default_availability: 0
|
||||
|
||||
# Deciders related to testing / debugging.
|
||||
enable_routing_to_ranker_dev_proxy:
|
||||
comment: "Route dark traffic to the TimelineRanker development proxy. 100% means ~100% of requests to a host."
|
||||
default_availability: 0
|
||||
|
||||
# Deciders related to authorization.
|
||||
client_request_authorization:
|
||||
comment: "Enable client request authorization and rate limiting"
|
||||
default_availability: 10000
|
||||
client_write_whitelist:
|
||||
comment: "Enable authorization of write protected requests from only whitelisted clients"
|
||||
default_availability: 0
|
||||
allow_timeline_mixer_recap_prod:
|
||||
comment: "Allow requests from production TimelineMixer/recap"
|
||||
default_availability: 10000
|
||||
allow_timeline_mixer_recycled_prod:
|
||||
comment: "Allow requests from production TimelineMixer/recycled"
|
||||
default_availability: 10000
|
||||
allow_timeline_mixer_hydrate_prod:
|
||||
comment: "Allow requests from production TimelineMixer/hydrate"
|
||||
default_availability: 10000
|
||||
allow_timeline_mixer_hydrate_recos_prod:
|
||||
comment: "Allow requests from production TimelineMixer/hydrate_recos"
|
||||
default_availability: 10000
|
||||
allow_timeline_mixer_seed_authors_prod:
|
||||
comment: "Allow requests from production TimelineMixer/seed_author_ids"
|
||||
default_availability: 10000
|
||||
allow_timeline_mixer_simcluster_prod:
|
||||
comment: "Allow requests from production TimelineMixer/simcluster"
|
||||
default_availability: 10000
|
||||
allow_timeline_mixer_entity_tweets_prod:
|
||||
comment: "Allow requests from production TimelineMixer/entity_tweets"
|
||||
default_availability: 10000
|
||||
allow_timeline_mixer_list_prod:
|
||||
comment: "Allow requests from production TimelineMixer/list"
|
||||
default_availability: 10000
|
||||
allow_timeline_mixer_list_tweet_prod:
|
||||
comment: "Allow requests from production TimelineMixer/list_tweet"
|
||||
default_availability: 10000
|
||||
allow_timeline_mixer_uteg_liked_by_tweets_prod:
|
||||
comment: "Allow requests from production TimelineMixer/uteg_liked_by_tweets"
|
||||
default_availability: 10000
|
||||
allow_timeline_mixer_community_prod:
|
||||
comment: "Allow requests from production TimelineMixer/community"
|
||||
default_availability: 10000
|
||||
allow_timeline_mixer_community_tweet_prod:
|
||||
comment: "Allow requests from production TimelineMixer/community_tweet"
|
||||
default_availability: 10000
|
||||
allow_timeline_scorer_recommended_trend_tweet_prod:
|
||||
comment: "Allow requests from production TimelineMixer/recommended_trend_tweet"
|
||||
default_availability: 10000
|
||||
|
||||
allow_timeline_scorer_rec_topic_tweets_prod:
|
||||
comment: "Allow requests from production TimelineScorer/rec_topic_tweets"
|
||||
default_availability: 10000
|
||||
allow_timeline_scorer_popular_topic_tweets_prod:
|
||||
comment: "Allow requests from production TimelineScorer/popular_topic_tweets"
|
||||
default_availability: 10000
|
||||
|
||||
allow_timelinescorer_hydrate_tweet_scoring_prod:
|
||||
comment: "Allow requests from production TimelineScorer/hydrate_tweet_scoring"
|
||||
default_availability: 10000
|
||||
|
||||
allow_timeline_mixer_staging:
|
||||
comment: "Allow requests from staging TimelineMixer"
|
||||
default_availability: 10000
|
||||
allow_timeline_ranker_warmup:
|
||||
comment: "Allow warmup requests from the TLR cluster"
|
||||
default_availability: 10000
|
||||
allow_timeline_ranker_proxy:
|
||||
comment: "Allow warmup requests from the TimelineRanker proxy"
|
||||
default_availability: 10000
|
||||
allow_timeline_service_prod:
|
||||
comment: "Allow requests from production TimelineService"
|
||||
default_availability: 10000
|
||||
allow_timeline_service_staging:
|
||||
comment: "Allow requests from staging TimelineService"
|
||||
default_availability: 10000
|
||||
rate_limit_override_unknown:
|
||||
comment: "Override the rate limit for unknown clients"
|
||||
default_availability: 0
|
||||
|
||||
# Deciders related to reverse-chron home timeline materialization.
|
||||
multiplier_of_materialization_tweets_fetched:
|
||||
comment: "Multiplier applied to the number of tweets fetched from search expressed as percentage. 100 means 100%. It can be used to fetch more than the number tweets requested by a caller (to improve similarity) or to fetch less than requested to reduce load."
|
||||
default_availability: 100
|
||||
enable_backfill_filtered_entries:
|
||||
comment: "Controls whether to back-fill timeline entries that get filtered out by TweetsPostFilter during home timeline materialization."
|
||||
default_availability: 0
|
||||
tweets_filtering_lossage_threshold:
|
||||
comment: "If back-filling filtered entries is enabled and if percentage of tweets that get filtered out exceeds this value then we will issue a second call to get more tweets. Default value 2000 == 20%"
|
||||
default_availability: 2000
|
||||
tweets_filtering_lossage_limit:
|
||||
comment: "We need to ensure that the number of tweets requested by the second call are not unbounded (for example, if everything is filtered out in the first call) therefore we limit the actual filtered out percentage to be no greater than the value below. Default value 6000 == 60%. That is, even if the actual lossage is 90% we will consider it to be only 60% for the purpose of back-filling."
|
||||
default_availability: 6000
|
||||
supplement_follows_with_real_graph:
|
||||
comment: "Whether to fetch additional follows from RealGraph for users with more than the max follows fetched from SGS during home timeline materialization."
|
||||
default_availability: 0
|
||||
|
||||
# Deciders related to recap.
|
||||
recap_enable_content_features_hydration:
|
||||
comment: "If true, semantic core, penguin, and tweetypie based expensive features will be hydrated for recap Tweets. Otherwise those features are not set"
|
||||
default_availability: 10000
|
||||
recap_max_count_multiplier:
|
||||
comment: "We multiply maxCount (caller supplied value) by this multiplier and fetch those many candidates from search so that we are left with sufficient number of candidates after hydration and filtering. 100 == 1.0"
|
||||
default_availability: 100
|
||||
recap_enable_extra_sorting_in_results:
|
||||
comment: "If TLR will do extra sorting in search results"
|
||||
default_availability: 10000
|
||||
|
||||
# Deciders related to recycled tweets.
|
||||
recycled_enable_content_features_hydration:
|
||||
comment: "If true, semantic core, penguin, and tweetypie based expensive features will be hydrated for recycled Tweets. Otherwise those features are not set"
|
||||
default_availability: 0
|
||||
recycled_max_count_multiplier:
|
||||
comment: "We multiply maxCount (caller supplied value) by this multiplier and fetch those many candidates from search so that we are left with sufficient number of candidates after hydration and filtering. 100 == 1.0"
|
||||
default_availability: 100
|
||||
|
||||
# Deciders related to entity tweets.
|
||||
entity_tweets_enable_content_features_hydration:
|
||||
comment: "If true, semantic core, penguin, and tweetypie based expensive features will be hydrated for entity Tweets. Otherwise those features are not set"
|
||||
default_availability: 10000
|
||||
|
||||
# Deciders related to both recap and recycled tweets
|
||||
enable_real_graph_users:
|
||||
comment: "This is used only if user follows >= 1000. If true, expands user seedset with real graph users and recent followed users. Otherwise, user seedset only includes followed users."
|
||||
default_availability: 0
|
||||
max_real_graph_and_followed_users:
|
||||
comment: "Maximum number of combined real graph users and recent followed users in the user seedset for recap and recycled tweets if enable_real_graph_users is true and only_real_graph_users is false. This is upper bounded by 2000."
|
||||
default_availability: 1000
|
||||
|
||||
# Deciders related to recap author
|
||||
recap_author_enable_new_pipeline:
|
||||
comment: "Enable new recap author pipeline"
|
||||
default_availability: 0
|
||||
recap_author_enable_content_features_hydration:
|
||||
comment: "If true, semantic core, penguin, and tweetypie based expensive features will be hydrated for PYLE Tweets. Otherwise those features are not set"
|
||||
default_availability: 0
|
||||
|
||||
# Deciders related to recap hydration(rectweet+ranked organic).
|
||||
recap_hydration_enable_content_features_hydration:
|
||||
comment: "If true, semantic core, penguin, and tweetypie based expensive features will be hydrated for rectweet+ranked organic Tweets. Otherwise those features are not set"
|
||||
default_availability: 0
|
||||
|
||||
# Deciders related to uteg liked by tweets
|
||||
uteg_liked_by_tweets_enable_content_features_hydration:
|
||||
comment: "If true, semantic core, penguin, and tweetypie based expensive features will be hydrated for rectweet+recycled utegLikedBy Tweets. Otherwise those features are not set"
|
||||
default_availability: 0
|
||||
5
timelineranker/server/src/main/resources/BUILD.bazel
Normal file
5
timelineranker/server/src/main/resources/BUILD.bazel
Normal file
@ -0,0 +1,5 @@
|
||||
resources(
|
||||
sources = [
|
||||
"*.xml",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,124 @@
|
||||
<configuration>
|
||||
<shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
|
||||
<property name="async_queue_size" value="${queue.size:-50000}"/>
|
||||
<property name="async_max_flush_time" value="${max.flush.time:-0}"/>
|
||||
<property name="SERVICE_OUTPUT" value="${log.service.output:-server.log}"/>
|
||||
<property name="DEBUG_TRANSCRIPTS_OUTPUT"
|
||||
value="${log.debug_transcripts.output:-debug_transcripts.log}"/>
|
||||
<property name="DEFAULT_SERVICE_PATTERN"
|
||||
value="%5p [%d{yyyyMMdd-HH:mm:ss.SSS}] %logger{0}: %m%n"/>
|
||||
<!-- JUL/JDK14 to Logback bridge -->
|
||||
<contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
|
||||
<resetJUL>true</resetJUL>
|
||||
</contextListener>
|
||||
|
||||
<!-- Service Log -->
|
||||
<appender name="SERVICE" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${SERVICE_OUTPUT}</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
|
||||
<!-- daily rollover -->
|
||||
<fileNamePattern>${SERVICE_OUTPUT}.%d.%i.gz</fileNamePattern>
|
||||
<maxFileSize>500MB</maxFileSize>
|
||||
<!-- keep 21 days' worth of history -->
|
||||
<maxHistory>21</maxHistory>
|
||||
<cleanHistoryOnStart>true</cleanHistoryOnStart>
|
||||
</rollingPolicy>
|
||||
<encoder>
|
||||
<pattern>${DEFAULT_SERVICE_PATTERN}</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<!-- debug transcripts -->
|
||||
<appender name="DEBUG-TRANSCRIPTS" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${DEBUG_TRANSCRIPTS_OUTPUT}</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
|
||||
<!-- daily rollover -->
|
||||
<fileNamePattern>${DEBUG_TRANSCRIPTS_OUTPUT}.%d.%i.gz</fileNamePattern>
|
||||
<maxFileSize>500MB</maxFileSize>
|
||||
<!-- keep 21 days' worth of history -->
|
||||
<maxHistory>21</maxHistory>
|
||||
<cleanHistoryOnStart>true</cleanHistoryOnStart>
|
||||
</rollingPolicy>
|
||||
<encoder>
|
||||
<pattern>${DEFAULT_SERVICE_PATTERN}</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<!-- LogLens/splunk -->
|
||||
<appender name="LOGLENS" class="com.twitter.loglens.logback.LoglensAppender">
|
||||
<mdcAdditionalContext>true</mdcAdditionalContext>
|
||||
<category>loglens</category>
|
||||
<index>${log.lens.index:-timelineranker}</index>
|
||||
<tag>${log.lens.tag}</tag>
|
||||
<encoder>
|
||||
<pattern>%msg%n</pattern>
|
||||
</encoder>
|
||||
<filter class="com.twitter.strato.logging.logback.RegexFilter">
|
||||
<forLogger>manhattan-client</forLogger>
|
||||
<excludeRegex>.*InvalidRequest.*</excludeRegex>
|
||||
</filter>
|
||||
</appender>
|
||||
|
||||
<!-- ===================================================== -->
|
||||
<!-- Primary Async Appenders -->
|
||||
<!-- ===================================================== -->
|
||||
|
||||
<appender name="ASYNC-SERVICE" class="ch.qos.logback.classic.AsyncAppender">
|
||||
<queueSize>${async_queue_size}</queueSize>
|
||||
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
|
||||
<appender-ref ref="SERVICE"/>
|
||||
</appender>
|
||||
|
||||
<appender name="ASYNC-DEBUG-TRANSCRIPTS" class="ch.qos.logback.classic.AsyncAppender">
|
||||
<queueSize>${async_queue_size}</queueSize>
|
||||
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
|
||||
<appender-ref ref="DEBUG-TRANSCRIPTS"/>
|
||||
</appender>
|
||||
|
||||
<appender name="ASYNC-LOGLENS" class="ch.qos.logback.classic.AsyncAppender">
|
||||
<queueSize>${async_queue_size}</queueSize>
|
||||
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
|
||||
<appender-ref ref="LOGLENS"/>
|
||||
</appender>
|
||||
|
||||
<!-- ===================================================== -->
|
||||
<!-- Package Config -->
|
||||
<!-- ===================================================== -->
|
||||
|
||||
<!-- Per-Package Config -->
|
||||
<logger name="OptimisticLockingCache" level="off"/>
|
||||
<logger name="ZkSession" level="info"/>
|
||||
<logger name="com.twitter" level="info"/>
|
||||
<logger name="com.twitter.decider.StoreDecider" level="warn"/>
|
||||
<logger name="com.twitter.distributedlog.client" level="warn"/>
|
||||
<logger name="com.twitter.finagle.liveness" level="warn"/>
|
||||
<logger name="com.twitter.finagle.mtls.authorization.config.AccessControlListConfiguration" level="warn"/>
|
||||
<logger name="com.twitter.finagle.mux" level="warn"/>
|
||||
<logger name="com.twitter.finagle.serverset2" level="warn"/>
|
||||
<logger name="com.twitter.finatra.kafka.common.kerberoshelpers" level="warn"/>
|
||||
<logger name="com.twitter.finatra.kafka.utils.BootstrapServerUtils" level="warn"/>
|
||||
<logger name="com.twitter.logging.ScribeHandler" level="warn"/>
|
||||
<logger name="com.twitter.server.coordinate" level="error"/>
|
||||
<logger name="com.twitter.wilyns" level="warn"/>
|
||||
<logger name="com.twitter.zookeeper.client" level="info"/>
|
||||
<logger name="com.twitter.zookeeper.client.internal" level="warn"/>
|
||||
<logger name="manhattan-client" level="warn"/>
|
||||
<logger name="org.apache.kafka.clients.NetworkClient" level="error"/>
|
||||
<logger name="org.apache.kafka.clients.consumer.internals" level="error"/>
|
||||
<logger name="org.apache.kafka.clients.producer.internals" level="error"/>
|
||||
<logger name="org.apache.kafka.common.network" level="warn"/>
|
||||
<logger name="org.apache.zookeeper" level="error"/>
|
||||
<logger name="org.apache.zookeeper.ClientCnxn" level="warn"/>
|
||||
|
||||
<!-- Root Config -->
|
||||
<root level="${log_level:-INFO}">
|
||||
<appender-ref ref="ASYNC-SERVICE"/>
|
||||
<appender-ref ref="ASYNC-LOGLENS"/>
|
||||
</root>
|
||||
|
||||
<!-- debug transcripts: logger name MUST be c.t.timelines.util.debuglog.DebugLog.DebugTranscriptsLog -->
|
||||
<logger name="DebugTranscripts" level="info">
|
||||
<appender-ref ref="ASYNC-DEBUG-TRANSCRIPTS"/>
|
||||
<appender-ref ref="ASYNC-LOGLENS"/>
|
||||
</logger>
|
||||
</configuration>
|
||||
32
timelineranker/server/src/main/scala/BUILD.bazel
Normal file
32
timelineranker/server/src/main/scala/BUILD.bazel
Normal file
@ -0,0 +1,32 @@
|
||||
target(
|
||||
dependencies = [
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/repository",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/server",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/source",
|
||||
],
|
||||
)
|
||||
|
||||
jvm_binary(
|
||||
name = "bin",
|
||||
basename = "timelineranker-server",
|
||||
main = "com.twitter.timelineranker.server.Main",
|
||||
runtime_platform = "java11",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
":scala",
|
||||
"3rdparty/jvm/org/slf4j:jcl-over-slf4j", # [1]
|
||||
"3rdparty/jvm/org/slf4j:log4j-over-slf4j", # [1]
|
||||
"loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback", # [2]
|
||||
"strato/src/main/scala/com/twitter/strato/logging/logback", # [2]
|
||||
"timelineranker/server/src/main/resources", # [2]
|
||||
"twitter-server/logback-classic/src/main/scala", #[2]
|
||||
],
|
||||
)
|
||||
|
||||
# [1] bridge other logging implementations to slf4j-api in addition to JUL
|
||||
# https://docbird.twitter.biz/core_libraries_guide/logging/twitter_server.html
|
||||
# without these, c.t.l.Logger become silent/null logger since no proper
|
||||
# configuration can be found. This can be removed once there are no
|
||||
# depdency from service to c.t.l.Logger
|
||||
#
|
||||
# [2] incur logback implementation
|
||||
@ -0,0 +1,22 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"cortex-core/thrift/src/main/thrift:thrift-scala",
|
||||
"cortex-tweet-annotate/service/src/main/thrift:thrift-scala",
|
||||
"finagle/finagle-memcached/src/main/scala",
|
||||
"mediaservices/commons/src/main/thrift:thrift-scala",
|
||||
"servo/repo",
|
||||
"servo/util/src/main/scala",
|
||||
"src/thrift/com/twitter/ml/api:data-scala",
|
||||
"src/thrift/com/twitter/ml/prediction_service:prediction_service-scala",
|
||||
"timelines/src/main/scala/com/twitter/timelines/model/types",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util/stats",
|
||||
"util/util-core:util-core-util",
|
||||
"util/util-logging/src/main/scala",
|
||||
"util/util-stats/src/main/scala",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,113 @@
|
||||
package com.twitter.timelineranker.clients
|
||||
|
||||
import com.twitter.cortex_core.thriftscala.ModelName
|
||||
import com.twitter.cortex_tweet_annotate.thriftscala._
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.logging.Logger
|
||||
import com.twitter.mediaservices.commons.mediainformation.thriftscala.CalibrationLevel
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.timelines.util.stats.RequestScope
|
||||
import com.twitter.timelines.util.stats.RequestStats
|
||||
import com.twitter.timelines.util.stats.ScopedFactory
|
||||
import com.twitter.timelines.util.FailOpenHandler
|
||||
import com.twitter.util.Future
|
||||
|
||||
object CortexTweetQueryServiceClient {
|
||||
private[this] val logger = Logger.get(getClass.getSimpleName)
|
||||
|
||||
/**
|
||||
* A tweet is considered safe if Cortex NSFA model gives it a score that is above the threshold.
|
||||
* Both the score and the threshold are returned in a response from getTweetSignalByIds endpoint.
|
||||
*/
|
||||
private def getSafeTweet(
|
||||
request: TweetSignalRequest,
|
||||
response: ModelResponseResult
|
||||
): Option[TweetId] = {
|
||||
val tweetId = request.tweetId
|
||||
response match {
|
||||
case ModelResponseResult(ModelResponseState.Success, Some(tid), Some(modelResponse), _) =>
|
||||
val prediction = modelResponse.predictions.flatMap(_.headOption)
|
||||
val score = prediction.map(_.score.score)
|
||||
val highRecallBucket = prediction.flatMap(_.calibrationBuckets).flatMap { buckets =>
|
||||
buckets.find(_.description.contains(CalibrationLevel.HighRecall))
|
||||
}
|
||||
val threshold = highRecallBucket.map(_.threshold)
|
||||
(score, threshold) match {
|
||||
case (Some(s), Some(t)) if (s > t) =>
|
||||
Some(tid)
|
||||
case (Some(s), Some(t)) =>
|
||||
logger.ifDebug(
|
||||
s"Cortex NSFA score for tweet $tweetId is $s (threshold is $t), removing as unsafe."
|
||||
)
|
||||
None
|
||||
case _ =>
|
||||
logger.ifDebug(s"Unexpected response, removing tweet $tweetId as unsafe.")
|
||||
None
|
||||
}
|
||||
case _ =>
|
||||
logger.ifWarning(
|
||||
s"Cortex tweet NSFA call was not successful, removing tweet $tweetId as unsafe."
|
||||
)
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enables calling cortex tweet query service to get NSFA scores on the tweet.
|
||||
*/
|
||||
class CortexTweetQueryServiceClient(
|
||||
cortexClient: CortexTweetQueryService.MethodPerEndpoint,
|
||||
requestScope: RequestScope,
|
||||
statsReceiver: StatsReceiver)
|
||||
extends RequestStats {
|
||||
import CortexTweetQueryServiceClient._
|
||||
|
||||
private[this] val logger = Logger.get(getClass.getSimpleName)
|
||||
|
||||
private[this] val getTweetSignalByIdsRequestStats =
|
||||
requestScope.stats("cortex", statsReceiver, suffix = Some("getTweetSignalByIds"))
|
||||
private[this] val getTweetSignalByIdsRequestScopedStatsReceiver =
|
||||
getTweetSignalByIdsRequestStats.scopedStatsReceiver
|
||||
|
||||
private[this] val failedCortexTweetQueryServiceScope =
|
||||
getTweetSignalByIdsRequestScopedStatsReceiver.scope(Failures)
|
||||
private[this] val failedCortexTweetQueryServiceCallCounter =
|
||||
failedCortexTweetQueryServiceScope.counter("failOpen")
|
||||
|
||||
private[this] val cortexTweetQueryServiceFailOpenHandler = new FailOpenHandler(
|
||||
getTweetSignalByIdsRequestScopedStatsReceiver
|
||||
)
|
||||
|
||||
def getSafeTweets(tweetIds: Seq[TweetId]): Future[Seq[TweetId]] = {
|
||||
val requests = tweetIds.map { id => TweetSignalRequest(id, ModelName.TweetToNsfa) }
|
||||
val results = cortexClient
|
||||
.getTweetSignalByIds(
|
||||
GetTweetSignalByIdsRequest(requests)
|
||||
)
|
||||
.map(_.results)
|
||||
|
||||
cortexTweetQueryServiceFailOpenHandler(
|
||||
results.map { responses =>
|
||||
requests.zip(responses).flatMap {
|
||||
case (request, response) =>
|
||||
getSafeTweet(request, response)
|
||||
}
|
||||
}
|
||||
) { _ =>
|
||||
failedCortexTweetQueryServiceCallCounter.incr()
|
||||
logger.ifWarning(s"Cortex tweet NSFA call failed, considering tweets $tweetIds as unsafe.")
|
||||
Future.value(Seq())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class ScopedCortexTweetQueryServiceClientFactory(
|
||||
cortexClient: CortexTweetQueryService.MethodPerEndpoint,
|
||||
statsReceiver: StatsReceiver)
|
||||
extends ScopedFactory[CortexTweetQueryServiceClient] {
|
||||
|
||||
override def scope(scope: RequestScope): CortexTweetQueryServiceClient = {
|
||||
new CortexTweetQueryServiceClient(cortexClient, scope, statsReceiver)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,48 @@
|
||||
package com.twitter.timelineranker.clients
|
||||
|
||||
import com.twitter.finagle.memcached.{Client => FinagleMemcacheClient}
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.logging.Logger
|
||||
import com.twitter.servo.cache.FinagleMemcache
|
||||
import com.twitter.servo.cache.MemcacheCache
|
||||
import com.twitter.servo.cache.ObservableMemcache
|
||||
import com.twitter.servo.cache.Serializer
|
||||
import com.twitter.servo.cache.StatsReceiverCacheObserver
|
||||
import com.twitter.timelines.util.stats.RequestScope
|
||||
import com.twitter.timelines.util.stats.ScopedFactory
|
||||
import com.twitter.util.Duration
|
||||
|
||||
/**
|
||||
* Factory to create a servo Memcache-backed Cache object. Clients are required to provide a
|
||||
* serializer/deserializer for keys and values.
|
||||
*/
|
||||
class MemcacheFactory(memcacheClient: FinagleMemcacheClient, statsReceiver: StatsReceiver) {
|
||||
private[this] val logger = Logger.get(getClass.getSimpleName)
|
||||
|
||||
def apply[K, V](
|
||||
keySerializer: K => String,
|
||||
valueSerializer: Serializer[V],
|
||||
ttl: Duration
|
||||
): MemcacheCache[K, V] = {
|
||||
new MemcacheCache[K, V](
|
||||
memcache = new ObservableMemcache(
|
||||
new FinagleMemcache(memcacheClient),
|
||||
new StatsReceiverCacheObserver(statsReceiver, 1000, logger)
|
||||
),
|
||||
ttl = ttl,
|
||||
serializer = valueSerializer,
|
||||
transformKey = keySerializer
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
class ScopedMemcacheFactory(memcacheClient: FinagleMemcacheClient, statsReceiver: StatsReceiver)
|
||||
extends ScopedFactory[MemcacheFactory] {
|
||||
|
||||
override def scope(scope: RequestScope): MemcacheFactory = {
|
||||
new MemcacheFactory(
|
||||
memcacheClient,
|
||||
statsReceiver.scope("memcache", scope.scope)
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,24 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
platform = "java8",
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/twitter/algebird:bijection",
|
||||
"3rdparty/jvm/com/twitter/bijection:core",
|
||||
"3rdparty/jvm/com/twitter/bijection:netty",
|
||||
"3rdparty/jvm/com/twitter/bijection:scrooge",
|
||||
"3rdparty/jvm/com/twitter/bijection:thrift",
|
||||
"3rdparty/jvm/com/twitter/bijection:util",
|
||||
"3rdparty/jvm/com/twitter/storehaus:core",
|
||||
"finagle/finagle-stats",
|
||||
"scrooge/scrooge-core/src/main/scala",
|
||||
"src/scala/com/twitter/summingbird_internal/bijection:bijection-implicits",
|
||||
"src/thrift/com/twitter/timelines/content_features:thrift-scala",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/memcache_common",
|
||||
"timelines/src/main/scala/com/twitter/timelines/model/types",
|
||||
"util/util-core:util-core-util",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,39 @@
|
||||
package com.twitter.timelineranker.clients.content_features_cache
|
||||
|
||||
import com.twitter.bijection.Injection
|
||||
import com.twitter.bijection.scrooge.CompactScalaCodec
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.storehaus.Store
|
||||
import com.twitter.timelineranker.recap.model.ContentFeatures
|
||||
import com.twitter.timelines.clients.memcache_common._
|
||||
import com.twitter.timelines.content_features.{thriftscala => thrift}
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.util.Duration
|
||||
|
||||
/**
|
||||
* Content features will be stored by tweetId
|
||||
*/
|
||||
class ContentFeaturesMemcacheBuilder(
|
||||
config: StorehausMemcacheConfig,
|
||||
ttl: Duration,
|
||||
statsReceiver: StatsReceiver) {
|
||||
private[this] val scalaToThriftInjection: Injection[ContentFeatures, thrift.ContentFeatures] =
|
||||
Injection.build[ContentFeatures, thrift.ContentFeatures](_.toThrift)(
|
||||
ContentFeatures.tryFromThrift)
|
||||
|
||||
private[this] val thriftToBytesInjection: Injection[thrift.ContentFeatures, Array[Byte]] =
|
||||
CompactScalaCodec(thrift.ContentFeatures)
|
||||
|
||||
private[this] implicit val valueInjection: Injection[ContentFeatures, Array[Byte]] =
|
||||
scalaToThriftInjection.andThen(thriftToBytesInjection)
|
||||
|
||||
private[this] val underlyingBuilder =
|
||||
new MemcacheStoreBuilder[TweetId, ContentFeatures](
|
||||
config = config,
|
||||
scopeName = "contentFeaturesCache",
|
||||
statsReceiver = statsReceiver,
|
||||
ttl = ttl
|
||||
)
|
||||
|
||||
def build(): Store[TweetId, ContentFeatures] = underlyingBuilder.build()
|
||||
}
|
||||
@ -0,0 +1,43 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/twitter/storehaus:core",
|
||||
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
|
||||
"finagle/finagle-core/src/main",
|
||||
"servo/util/src/main/scala",
|
||||
"src/thrift/com/twitter/search:earlybird-scala",
|
||||
"src/thrift/com/twitter/search/common:constants-scala",
|
||||
"src/thrift/com/twitter/search/common:features-scala",
|
||||
"src/thrift/com/twitter/service/metastore/gen:thrift-scala",
|
||||
"timelineranker/common/src/main/scala/com/twitter/timelineranker/model",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/contentfeatures",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/core",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/util",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/visibility",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/manhattan",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/relevance_search",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/tweetypie",
|
||||
"timelines/src/main/scala/com/twitter/timelines/common/model",
|
||||
"timelines/src/main/scala/com/twitter/timelines/earlybird/common/options",
|
||||
"timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils",
|
||||
"timelines/src/main/scala/com/twitter/timelines/model/candidate",
|
||||
"timelines/src/main/scala/com/twitter/timelines/model/tweet",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util/bounds",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util/stats",
|
||||
"timelines/src/main/scala/com/twitter/timelines/visibility",
|
||||
"timelines/src/main/scala/com/twitter/timelines/visibility/model",
|
||||
"util/util-core:util-core-util",
|
||||
"util/util-core/src/main/scala/com/twitter/conversions",
|
||||
"util/util-logging/src/main/scala/com/twitter/logging",
|
||||
"util/util-stats/src/main/scala",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,40 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope
|
||||
import com.twitter.timelineranker.model.CandidateTweet
|
||||
import com.twitter.timelineranker.model.CandidateTweetsResult
|
||||
import com.twitter.util.Future
|
||||
|
||||
class CandidateGenerationTransform(statsReceiver: StatsReceiver)
|
||||
extends FutureArrow[HydratedCandidatesAndFeaturesEnvelope, CandidateTweetsResult] {
|
||||
private[this] val numCandidateTweetsStat = statsReceiver.stat("numCandidateTweets")
|
||||
private[this] val numSourceTweetsStat = statsReceiver.stat("numSourceTweets")
|
||||
|
||||
override def apply(
|
||||
candidatesAndFeaturesEnvelope: HydratedCandidatesAndFeaturesEnvelope
|
||||
): Future[CandidateTweetsResult] = {
|
||||
val hydratedTweets = candidatesAndFeaturesEnvelope.candidateEnvelope.hydratedTweets.outerTweets
|
||||
|
||||
if (hydratedTweets.nonEmpty) {
|
||||
val candidates = hydratedTweets.map { hydratedTweet =>
|
||||
CandidateTweet(hydratedTweet, candidatesAndFeaturesEnvelope.features(hydratedTweet.tweetId))
|
||||
}
|
||||
numCandidateTweetsStat.add(candidates.size)
|
||||
|
||||
val sourceTweets =
|
||||
candidatesAndFeaturesEnvelope.candidateEnvelope.sourceHydratedTweets.outerTweets.map {
|
||||
hydratedTweet =>
|
||||
CandidateTweet(
|
||||
hydratedTweet,
|
||||
candidatesAndFeaturesEnvelope.features(hydratedTweet.tweetId))
|
||||
}
|
||||
numSourceTweetsStat.add(sourceTweets.size)
|
||||
|
||||
Future.value(CandidateTweetsResult(candidates, sourceTweets))
|
||||
} else {
|
||||
Future.value(CandidateTweetsResult.Empty)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,112 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.servo.util.Gate
|
||||
import com.twitter.storehaus.Store
|
||||
import com.twitter.timelineranker.contentfeatures.ContentFeaturesProvider
|
||||
import com.twitter.timelineranker.core.FutureDependencyTransformer
|
||||
import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery
|
||||
import com.twitter.timelineranker.recap.model.ContentFeatures
|
||||
import com.twitter.timelineranker.util.SearchResultUtil._
|
||||
import com.twitter.timelineranker.util.CachingContentFeaturesProvider
|
||||
import com.twitter.timelineranker.util.TweetHydrator
|
||||
import com.twitter.timelineranker.util.TweetypieContentFeaturesProvider
|
||||
import com.twitter.timelines.clients.tweetypie.TweetyPieClient
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.util.Future
|
||||
import com.twitter.timelines.configapi
|
||||
import com.twitter.timelines.util.FutureUtils
|
||||
|
||||
class ContentFeaturesHydrationTransformBuilder(
|
||||
tweetyPieClient: TweetyPieClient,
|
||||
contentFeaturesCache: Store[TweetId, ContentFeatures],
|
||||
enableContentFeaturesGate: Gate[RecapQuery],
|
||||
enableTokensInContentFeaturesGate: Gate[RecapQuery],
|
||||
enableTweetTextInContentFeaturesGate: Gate[RecapQuery],
|
||||
enableConversationControlContentFeaturesGate: Gate[RecapQuery],
|
||||
enableTweetMediaHydrationGate: Gate[RecapQuery],
|
||||
hydrateInReplyToTweets: Boolean,
|
||||
statsReceiver: StatsReceiver) {
|
||||
val scopedStatsReceiver: StatsReceiver = statsReceiver.scope("ContentFeaturesHydrationTransform")
|
||||
val tweetHydrator: TweetHydrator = new TweetHydrator(tweetyPieClient, scopedStatsReceiver)
|
||||
val tweetypieContentFeaturesProvider: ContentFeaturesProvider =
|
||||
new TweetypieContentFeaturesProvider(
|
||||
tweetHydrator,
|
||||
enableContentFeaturesGate,
|
||||
enableTokensInContentFeaturesGate,
|
||||
enableTweetTextInContentFeaturesGate,
|
||||
enableConversationControlContentFeaturesGate,
|
||||
enableTweetMediaHydrationGate,
|
||||
scopedStatsReceiver
|
||||
)
|
||||
|
||||
val cachingContentFeaturesProvider: ContentFeaturesProvider = new CachingContentFeaturesProvider(
|
||||
underlying = tweetypieContentFeaturesProvider,
|
||||
contentFeaturesCache = contentFeaturesCache,
|
||||
statsReceiver = scopedStatsReceiver
|
||||
)
|
||||
|
||||
val contentFeaturesProvider: configapi.FutureDependencyTransformer[RecapQuery, Seq[TweetId], Map[
|
||||
TweetId,
|
||||
ContentFeatures
|
||||
]] = FutureDependencyTransformer.partition(
|
||||
gate = enableContentFeaturesGate,
|
||||
ifTrue = cachingContentFeaturesProvider,
|
||||
ifFalse = tweetypieContentFeaturesProvider
|
||||
)
|
||||
|
||||
lazy val contentFeaturesHydrationTransform: ContentFeaturesHydrationTransform =
|
||||
new ContentFeaturesHydrationTransform(
|
||||
contentFeaturesProvider,
|
||||
enableContentFeaturesGate,
|
||||
hydrateInReplyToTweets
|
||||
)
|
||||
def build(): ContentFeaturesHydrationTransform = contentFeaturesHydrationTransform
|
||||
}
|
||||
|
||||
class ContentFeaturesHydrationTransform(
|
||||
contentFeaturesProvider: ContentFeaturesProvider,
|
||||
enableContentFeaturesGate: Gate[RecapQuery],
|
||||
hydrateInReplyToTweets: Boolean)
|
||||
extends FutureArrow[
|
||||
HydratedCandidatesAndFeaturesEnvelope,
|
||||
HydratedCandidatesAndFeaturesEnvelope
|
||||
] {
|
||||
override def apply(
|
||||
request: HydratedCandidatesAndFeaturesEnvelope
|
||||
): Future[HydratedCandidatesAndFeaturesEnvelope] = {
|
||||
if (enableContentFeaturesGate(request.candidateEnvelope.query)) {
|
||||
val searchResults = request.candidateEnvelope.searchResults
|
||||
|
||||
val sourceTweetIdMap = searchResults.map { searchResult =>
|
||||
(searchResult.id, getRetweetSourceTweetId(searchResult).getOrElse(searchResult.id))
|
||||
}.toMap
|
||||
|
||||
val inReplyToTweetIds = if (hydrateInReplyToTweets) {
|
||||
searchResults.flatMap(getInReplyToTweetId)
|
||||
} else {
|
||||
Seq.empty
|
||||
}
|
||||
|
||||
val tweetIdsToHydrate = (sourceTweetIdMap.values ++ inReplyToTweetIds).toSeq.distinct
|
||||
|
||||
val contentFeaturesMapFuture = if (tweetIdsToHydrate.nonEmpty) {
|
||||
contentFeaturesProvider(request.candidateEnvelope.query, tweetIdsToHydrate)
|
||||
} else {
|
||||
FutureUtils.EmptyMap[TweetId, ContentFeatures]
|
||||
}
|
||||
|
||||
Future.value(
|
||||
request.copy(
|
||||
contentFeaturesFuture = contentFeaturesMapFuture,
|
||||
tweetSourceTweetMap = sourceTweetIdMap,
|
||||
inReplyToTweetIds = inReplyToTweetIds.toSet
|
||||
)
|
||||
)
|
||||
} else {
|
||||
Future.value(request)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,15 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* Create a CandidateEnvelope based on the incoming RecapQuery
|
||||
*/
|
||||
object CreateCandidateEnvelopeTransform extends FutureArrow[RecapQuery, CandidateEnvelope] {
|
||||
override def apply(query: RecapQuery): Future[CandidateEnvelope] = {
|
||||
Future.value(CandidateEnvelope(query))
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,33 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* Fetches all data required for feature hydration and generates the HydratedCandidatesAndFeaturesEnvelope
|
||||
* @param tweetHydrationAndFilteringPipeline Pipeline which fetches the candidate tweets, hydrates and filters them
|
||||
* @param languagesService Fetch user languages, required for feature hydration
|
||||
* @param userProfileInfoService Fetch user profile info, required for feature hydration
|
||||
*/
|
||||
class FeatureHydrationDataTransform(
|
||||
tweetHydrationAndFilteringPipeline: FutureArrow[RecapQuery, CandidateEnvelope],
|
||||
languagesService: UserLanguagesTransform,
|
||||
userProfileInfoService: UserProfileInfoTransform)
|
||||
extends FutureArrow[RecapQuery, HydratedCandidatesAndFeaturesEnvelope] {
|
||||
override def apply(request: RecapQuery): Future[HydratedCandidatesAndFeaturesEnvelope] = {
|
||||
Future
|
||||
.join(
|
||||
languagesService(request),
|
||||
userProfileInfoService(request),
|
||||
tweetHydrationAndFilteringPipeline(request)).map {
|
||||
case (languages, userProfileInfo, transformedCandidateEnvelope) =>
|
||||
HydratedCandidatesAndFeaturesEnvelope(
|
||||
transformedCandidateEnvelope,
|
||||
languages,
|
||||
userProfileInfo)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,198 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.finagle.stats.Stat
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
|
||||
import com.twitter.timelineranker.parameters.recap.RecapQueryContext
|
||||
import com.twitter.timelineranker.parameters.in_network_tweets.InNetworkTweetParams.RecycledMaxFollowedUsersEnableAntiDilutionParam
|
||||
import com.twitter.timelineranker.visibility.FollowGraphDataProvider
|
||||
import com.twitter.timelines.earlybird.common.options.AuthorScoreAdjustments
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* Transform which conditionally augments follow graph data using the real graph,
|
||||
* derived from the earlybirdOptions passed in the query
|
||||
*
|
||||
* @param followGraphDataProvider data provider to be used for fetching updated mutual follow info
|
||||
* @param maxFollowedUsersProvider max number of users to return
|
||||
* @param enableRealGraphUsersProvider should we augment using real graph data?
|
||||
* @param maxRealGraphAndFollowedUsersProvider max combined users to return, overrides maxFollowedUsersProvider above
|
||||
* @param statsReceiver scoped stats received
|
||||
*/
|
||||
class FollowAndRealGraphCombiningTransform(
|
||||
followGraphDataProvider: FollowGraphDataProvider,
|
||||
maxFollowedUsersProvider: DependencyProvider[Int],
|
||||
enableRealGraphUsersProvider: DependencyProvider[Boolean],
|
||||
maxRealGraphAndFollowedUsersProvider: DependencyProvider[Int],
|
||||
imputeRealGraphAuthorWeightsProvider: DependencyProvider[Boolean],
|
||||
imputeRealGraphAuthorWeightsPercentileProvider: DependencyProvider[Int],
|
||||
statsReceiver: StatsReceiver)
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
|
||||
// Number of authors in the seedset after mixing followed users and real graph users
|
||||
// Only have this stat if user follows >= maxFollowedUsers and enableRealGraphUsers is true and onlyRealGraphUsers is false
|
||||
val numFollowAndRealGraphUsersStat: Stat = statsReceiver.stat("numFollowAndRealGraphUsers")
|
||||
val numFollowAndRealGraphUsersFromFollowGraphStat =
|
||||
statsReceiver.scope("numFollowAndRealGraphUsers").stat("FollowGraphUsers")
|
||||
val numFollowAndRealGraphUsersFromRealGraphStat =
|
||||
statsReceiver.scope("numFollowAndRealGraphUsers").stat("RealGraphUsers")
|
||||
val numFollowAndRealGraphUsersFromRealGraphCounter =
|
||||
statsReceiver.scope("numFollowAndRealGraphUsers").counter()
|
||||
|
||||
// Number of authors in the seedset with only followed users
|
||||
// Only have this stat if user follows >= maxFollowedUsers and enableRealGraphUsers is false
|
||||
val numFollowedUsersStat: Stat = statsReceiver.stat("numFollowedUsers")
|
||||
|
||||
// Number of authors in the seedset with only followed users
|
||||
// Only have this stat if user follows < maxFollowedUsers
|
||||
val numFollowedUsersLessThanMaxStat: Stat = statsReceiver.stat("numFollowedUsersLessThanMax")
|
||||
val numFollowedUsersLessThanMaxCounter =
|
||||
statsReceiver.scope("numFollowedUsersLessThanMax").counter()
|
||||
val numFollowedUsersMoreThanMaxStat: Stat = statsReceiver.stat("numFollowedUsersMoreThanMax")
|
||||
val numFollowedUsersMoreThanMaxCounter =
|
||||
statsReceiver.scope("numFollowedUsersMoreThanMax").counter()
|
||||
|
||||
val realGraphAuthorWeightsSumProdStat: Stat = statsReceiver.stat("realGraphAuthorWeightsSumProd")
|
||||
val realGraphAuthorWeightsSumMinExpStat: Stat =
|
||||
statsReceiver.stat("realGraphAuthorWeightsSumMinExp")
|
||||
val realGraphAuthorWeightsSumP50ExpStat: Stat =
|
||||
statsReceiver.stat("realGraphAuthorWeightsSumP50Exp")
|
||||
val realGraphAuthorWeightsSumP95ExpStat: Stat =
|
||||
statsReceiver.stat("realGraphAuthorWeightsSumP95Exp")
|
||||
val numAuthorsWithoutRealgraphScoreStat: Stat =
|
||||
statsReceiver.stat("numAuthorsWithoutRealgraphScore")
|
||||
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
val realGraphData = envelope.query.earlybirdOptions
|
||||
.map(_.authorScoreAdjustments.authorScoreMap)
|
||||
.getOrElse(Map.empty)
|
||||
|
||||
Future
|
||||
.join(
|
||||
envelope.followGraphData.followedUserIdsFuture,
|
||||
envelope.followGraphData.mutedUserIdsFuture
|
||||
).map {
|
||||
case (followedUserIds, mutedUserIds) =>
|
||||
// Anti-dilution param for DDG-16198
|
||||
val recycledMaxFollowedUsersEnableAntiDilutionParamProvider =
|
||||
DependencyProvider.from(RecycledMaxFollowedUsersEnableAntiDilutionParam)
|
||||
|
||||
val maxFollowedUsers = {
|
||||
if (followedUserIds.size > RecapQueryContext.MaxFollowedUsers.default && recycledMaxFollowedUsersEnableAntiDilutionParamProvider(
|
||||
envelope.query)) {
|
||||
// trigger experiment
|
||||
maxFollowedUsersProvider(envelope.query)
|
||||
} else {
|
||||
maxFollowedUsersProvider(envelope.query)
|
||||
}
|
||||
}
|
||||
|
||||
val filteredRealGraphUserIds = realGraphData.keySet
|
||||
.filterNot(mutedUserIds)
|
||||
.take(maxFollowedUsers)
|
||||
.toSeq
|
||||
|
||||
val filteredFollowedUserIds = followedUserIds.filterNot(mutedUserIds)
|
||||
|
||||
if (followedUserIds.size < maxFollowedUsers) {
|
||||
numFollowedUsersLessThanMaxStat.add(filteredFollowedUserIds.size)
|
||||
// stats
|
||||
numFollowedUsersLessThanMaxCounter.incr()
|
||||
(filteredFollowedUserIds, false)
|
||||
} else {
|
||||
numFollowedUsersMoreThanMaxStat.add(filteredFollowedUserIds.size)
|
||||
numFollowedUsersMoreThanMaxCounter.incr()
|
||||
if (enableRealGraphUsersProvider(envelope.query)) {
|
||||
val maxRealGraphAndFollowedUsersNum =
|
||||
maxRealGraphAndFollowedUsersProvider(envelope.query)
|
||||
require(
|
||||
maxRealGraphAndFollowedUsersNum >= maxFollowedUsers,
|
||||
"maxRealGraphAndFollowedUsers must be greater than or equal to maxFollowedUsers."
|
||||
)
|
||||
val recentFollowedUsersNum = RecapQueryContext.MaxFollowedUsers.bounds
|
||||
.apply(maxRealGraphAndFollowedUsersNum - filteredRealGraphUserIds.size)
|
||||
|
||||
val recentFollowedUsers =
|
||||
filteredFollowedUserIds
|
||||
.filterNot(filteredRealGraphUserIds.contains)
|
||||
.take(recentFollowedUsersNum)
|
||||
|
||||
val filteredFollowAndRealGraphUserIds =
|
||||
recentFollowedUsers ++ filteredRealGraphUserIds
|
||||
|
||||
// Track the size of recentFollowedUsers from SGS
|
||||
numFollowAndRealGraphUsersFromFollowGraphStat.add(recentFollowedUsers.size)
|
||||
// Track the size of filteredRealGraphUserIds from real graph dataset.
|
||||
numFollowAndRealGraphUsersFromRealGraphStat.add(filteredRealGraphUserIds.size)
|
||||
|
||||
numFollowAndRealGraphUsersFromRealGraphCounter.incr()
|
||||
|
||||
numFollowAndRealGraphUsersStat.add(filteredFollowAndRealGraphUserIds.size)
|
||||
|
||||
(filteredFollowAndRealGraphUserIds, true)
|
||||
} else {
|
||||
numFollowedUsersStat.add(followedUserIds.size)
|
||||
(filteredFollowedUserIds, false)
|
||||
}
|
||||
}
|
||||
}.map {
|
||||
case (updatedFollowSeq, shouldUpdateMutualFollows) =>
|
||||
val updatedMutualFollowing = if (shouldUpdateMutualFollows) {
|
||||
followGraphDataProvider.getMutuallyFollowingUserIds(
|
||||
envelope.query.userId,
|
||||
updatedFollowSeq)
|
||||
} else {
|
||||
envelope.followGraphData.mutuallyFollowingUserIdsFuture
|
||||
}
|
||||
|
||||
val followGraphData = envelope.followGraphData.copy(
|
||||
followedUserIdsFuture = Future.value(updatedFollowSeq),
|
||||
mutuallyFollowingUserIdsFuture = updatedMutualFollowing
|
||||
)
|
||||
|
||||
val authorIdsWithRealgraphScore = realGraphData.keySet
|
||||
val authorIdsWithoutRealgraphScores =
|
||||
updatedFollowSeq.filterNot(authorIdsWithRealgraphScore.contains)
|
||||
|
||||
//stat for logging the percentage of users' followings that do not have a realgraph score
|
||||
if (updatedFollowSeq.nonEmpty)
|
||||
numAuthorsWithoutRealgraphScoreStat.add(
|
||||
authorIdsWithoutRealgraphScores.size / updatedFollowSeq.size * 100)
|
||||
|
||||
if (imputeRealGraphAuthorWeightsProvider(envelope.query) && realGraphData.nonEmpty) {
|
||||
val imputedScorePercentile =
|
||||
imputeRealGraphAuthorWeightsPercentileProvider(envelope.query) / 100.0
|
||||
val existingAuthorIdScores = realGraphData.values.toList.sorted
|
||||
val imputedScoreIndex = Math.min(
|
||||
existingAuthorIdScores.length - 1,
|
||||
(existingAuthorIdScores.length * imputedScorePercentile).toInt)
|
||||
val imputedScore = existingAuthorIdScores(imputedScoreIndex)
|
||||
|
||||
val updatedAuthorScoreMap = realGraphData ++ authorIdsWithoutRealgraphScores
|
||||
.map(_ -> imputedScore).toMap
|
||||
imputedScorePercentile match {
|
||||
case 0.0 =>
|
||||
realGraphAuthorWeightsSumMinExpStat.add(updatedAuthorScoreMap.values.sum.toFloat)
|
||||
case 0.5 =>
|
||||
realGraphAuthorWeightsSumP50ExpStat.add(updatedAuthorScoreMap.values.sum.toFloat)
|
||||
case 0.95 =>
|
||||
realGraphAuthorWeightsSumP95ExpStat.add(updatedAuthorScoreMap.values.sum.toFloat)
|
||||
case _ =>
|
||||
}
|
||||
val earlybirdOptionsWithUpdatedAuthorScoreMap = envelope.query.earlybirdOptions
|
||||
.map(_.copy(authorScoreAdjustments = AuthorScoreAdjustments(updatedAuthorScoreMap)))
|
||||
val updatedQuery =
|
||||
envelope.query.copy(earlybirdOptions = earlybirdOptionsWithUpdatedAuthorScoreMap)
|
||||
envelope.copy(query = updatedQuery, followGraphData = followGraphData)
|
||||
} else {
|
||||
envelope.query.earlybirdOptions
|
||||
.map(_.authorScoreAdjustments.authorScoreMap.values.sum.toFloat).foreach {
|
||||
realGraphAuthorWeightsSumProdStat.add(_)
|
||||
}
|
||||
envelope.copy(followGraphData = followGraphData)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,23 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
|
||||
import com.twitter.timelineranker.visibility.FollowGraphDataProvider
|
||||
import com.twitter.util.Future
|
||||
|
||||
class FollowGraphDataTransform(
|
||||
followGraphDataProvider: FollowGraphDataProvider,
|
||||
maxFollowedUsersProvider: DependencyProvider[Int])
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
|
||||
val followGraphData = followGraphDataProvider.getAsync(
|
||||
envelope.query.userId,
|
||||
maxFollowedUsersProvider(envelope.query)
|
||||
)
|
||||
|
||||
Future.value(envelope.copy(followGraphData = followGraphData))
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,31 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* Transform that explicitly hydrates candidate tweets and fetches source tweets in parallel
|
||||
* and then joins the results back into the original Envelope
|
||||
* @param candidateTweetHydration Pipeline that hydrates candidate tweets
|
||||
* @param sourceTweetHydration Pipeline that fetches and hydrates source tweets
|
||||
*/
|
||||
class HydrateTweetsAndSourceTweetsInParallelTransform(
|
||||
candidateTweetHydration: FutureArrow[CandidateEnvelope, CandidateEnvelope],
|
||||
sourceTweetHydration: FutureArrow[CandidateEnvelope, CandidateEnvelope])
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
Future
|
||||
.join(
|
||||
candidateTweetHydration(envelope),
|
||||
sourceTweetHydration(envelope)
|
||||
).map {
|
||||
case (candidateTweetEnvelope, sourceTweetEnvelope) =>
|
||||
envelope.copy(
|
||||
hydratedTweets = candidateTweetEnvelope.hydratedTweets,
|
||||
sourceSearchResults = sourceTweetEnvelope.sourceSearchResults,
|
||||
sourceHydratedTweets = sourceTweetEnvelope.sourceHydratedTweets
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,96 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.logging.Logger
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.core.HydratedTweets
|
||||
import com.twitter.timelineranker.util.TweetFilters
|
||||
import com.twitter.timelineranker.util.TweetsPostFilter
|
||||
import com.twitter.timelines.model.UserId
|
||||
import com.twitter.util.Future
|
||||
|
||||
object HydratedTweetsFilterTransform {
|
||||
val EmptyFollowGraphDataTuple: (Seq[UserId], Seq[UserId], Set[UserId]) =
|
||||
(Seq.empty[UserId], Seq.empty[UserId], Set.empty[UserId])
|
||||
val DefaultNumRetweetsAllowed = 1
|
||||
|
||||
// Number of duplicate retweets (including the first one) allowed.
|
||||
// For example,
|
||||
// If there are 7 retweets of a given tweet, the following value will cause 5 of them
|
||||
// to be returned after filtering and the additional 2 will be filtered out.
|
||||
val NumDuplicateRetweetsAllowed = 5
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform which takes TweetFilters ValueSets for inner and outer tweets and uses
|
||||
* TweetsPostFilter to filter down the HydratedTweets using the supplied filters
|
||||
*
|
||||
* @param useFollowGraphData - use follow graph for filtering; otherwise only does filtering
|
||||
* independent of follow graph data
|
||||
* @param useSourceTweets - only needed when filtering extended replies
|
||||
* @param statsReceiver - scoped stats receiver
|
||||
*/
|
||||
class HydratedTweetsFilterTransform(
|
||||
outerFilters: TweetFilters.ValueSet,
|
||||
innerFilters: TweetFilters.ValueSet,
|
||||
useFollowGraphData: Boolean,
|
||||
useSourceTweets: Boolean,
|
||||
statsReceiver: StatsReceiver,
|
||||
numRetweetsAllowed: Int = HydratedTweetsFilterTransform.DefaultNumRetweetsAllowed)
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
import HydratedTweetsFilterTransform._
|
||||
|
||||
val logger: Logger = Logger.get(getClass.getSimpleName)
|
||||
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
if (outerFilters == TweetFilters.None) {
|
||||
Future.value(envelope)
|
||||
} else {
|
||||
val tweetsPostOuterFilter = new TweetsPostFilter(outerFilters, logger, statsReceiver)
|
||||
val tweetsPostInnerFilter = new TweetsPostFilter(innerFilters, logger, statsReceiver)
|
||||
|
||||
val graphData = if (useFollowGraphData) {
|
||||
Future.join(
|
||||
envelope.followGraphData.followedUserIdsFuture,
|
||||
envelope.followGraphData.inNetworkUserIdsFuture,
|
||||
envelope.followGraphData.mutedUserIdsFuture
|
||||
)
|
||||
} else {
|
||||
Future.value(EmptyFollowGraphDataTuple)
|
||||
}
|
||||
|
||||
val sourceTweets = if (useSourceTweets) {
|
||||
envelope.sourceHydratedTweets.outerTweets
|
||||
} else {
|
||||
Nil
|
||||
}
|
||||
|
||||
graphData.map {
|
||||
case (followedUserIds, inNetworkUserIds, mutedUserIds) =>
|
||||
val outerTweets = tweetsPostOuterFilter(
|
||||
userId = envelope.query.userId,
|
||||
followedUserIds = followedUserIds,
|
||||
inNetworkUserIds = inNetworkUserIds,
|
||||
mutedUserIds = mutedUserIds,
|
||||
tweets = envelope.hydratedTweets.outerTweets,
|
||||
numRetweetsAllowed = numRetweetsAllowed,
|
||||
sourceTweets = sourceTweets
|
||||
)
|
||||
val innerTweets = tweetsPostInnerFilter(
|
||||
userId = envelope.query.userId,
|
||||
followedUserIds = followedUserIds,
|
||||
inNetworkUserIds = inNetworkUserIds,
|
||||
mutedUserIds = mutedUserIds,
|
||||
// inner tweets refers to quoted tweets not source tweets, and special rulesets
|
||||
// in birdherd handle visibility of viewer to inner tweet author for these tweets.
|
||||
tweets = envelope.hydratedTweets.innerTweets,
|
||||
numRetweetsAllowed = numRetweetsAllowed,
|
||||
sourceTweets = sourceTweets
|
||||
)
|
||||
|
||||
envelope.copy(hydratedTweets = HydratedTweets(outerTweets, innerTweets))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope
|
||||
import com.twitter.timelines.earlybird.common.utils.EarlybirdFeaturesHydrator
|
||||
import com.twitter.util.Future
|
||||
|
||||
object InNetworkTweetsSearchFeaturesHydrationTransform
|
||||
extends FutureArrow[
|
||||
HydratedCandidatesAndFeaturesEnvelope,
|
||||
HydratedCandidatesAndFeaturesEnvelope
|
||||
] {
|
||||
override def apply(
|
||||
request: HydratedCandidatesAndFeaturesEnvelope
|
||||
): Future[HydratedCandidatesAndFeaturesEnvelope] = {
|
||||
Future
|
||||
.join(
|
||||
request.candidateEnvelope.followGraphData.followedUserIdsFuture,
|
||||
request.candidateEnvelope.followGraphData.mutuallyFollowingUserIdsFuture
|
||||
).map {
|
||||
case (followedIds, mutuallyFollowingIds) =>
|
||||
val featuresByTweetId = EarlybirdFeaturesHydrator.hydrate(
|
||||
searcherUserId = request.candidateEnvelope.query.userId,
|
||||
searcherProfileInfo = request.userProfileInfo,
|
||||
followedUserIds = followedIds,
|
||||
mutuallyFollowingUserIds = mutuallyFollowingIds,
|
||||
userLanguages = request.userLanguages,
|
||||
uiLanguageCode = request.candidateEnvelope.query.deviceContext.flatMap(_.languageCode),
|
||||
searchResults = request.candidateEnvelope.searchResults,
|
||||
sourceTweetSearchResults = request.candidateEnvelope.sourceSearchResults,
|
||||
tweets = request.candidateEnvelope.hydratedTweets.outerTweets,
|
||||
sourceTweets = request.candidateEnvelope.sourceHydratedTweets.outerTweets
|
||||
)
|
||||
|
||||
request.copy(features = featuresByTweetId)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,55 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.model.CandidateTweet
|
||||
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
|
||||
import com.twitter.util.Future
|
||||
import com.twitter.util.Time
|
||||
import scala.util.Random
|
||||
|
||||
/**
|
||||
* picks up one or more random tweets and sets its tweetFeatures.isRandomTweet field to true.
|
||||
*/
|
||||
class MarkRandomTweetTransform(
|
||||
includeRandomTweetProvider: DependencyProvider[Boolean],
|
||||
randomGenerator: Random = new Random(Time.now.inMilliseconds),
|
||||
includeSingleRandomTweetProvider: DependencyProvider[Boolean],
|
||||
probabilityRandomTweetProvider: DependencyProvider[Double])
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
val includeRandomTweet = includeRandomTweetProvider(envelope.query)
|
||||
val includeSingleRandomTweet = includeSingleRandomTweetProvider(envelope.query)
|
||||
val probabilityRandomTweet = probabilityRandomTweetProvider(envelope.query)
|
||||
val searchResults = envelope.searchResults
|
||||
|
||||
if (!includeRandomTweet || searchResults.isEmpty) { // random tweet off
|
||||
Future.value(envelope)
|
||||
} else if (includeSingleRandomTweet) { // pick only one
|
||||
val randomIdx = randomGenerator.nextInt(searchResults.size)
|
||||
val randomTweet = searchResults(randomIdx)
|
||||
val randomTweetWithFlag = randomTweet.copy(
|
||||
tweetFeatures = randomTweet.tweetFeatures
|
||||
.orElse(Some(CandidateTweet.DefaultFeatures))
|
||||
.map(_.copy(isRandomTweet = Some(true)))
|
||||
)
|
||||
val updatedSearchResults = searchResults.updated(randomIdx, randomTweetWithFlag)
|
||||
|
||||
Future.value(envelope.copy(searchResults = updatedSearchResults))
|
||||
} else { // pick tweets with perTweetProbability
|
||||
val updatedSearchResults = searchResults.map { result =>
|
||||
if (randomGenerator.nextDouble() < probabilityRandomTweet) {
|
||||
result.copy(
|
||||
tweetFeatures = result.tweetFeatures
|
||||
.orElse(Some(CandidateTweet.DefaultFeatures))
|
||||
.map(_.copy(isRandomTweet = Some(true))))
|
||||
|
||||
} else
|
||||
result
|
||||
}
|
||||
|
||||
Future.value(envelope.copy(searchResults = updatedSearchResults))
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,49 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelines.clients.relevance_search.SearchClient
|
||||
import com.twitter.util.Future
|
||||
|
||||
object OutOfNetworkRepliesToUserIdSearchResultsTransform {
|
||||
val DefaultMaxTweetCount = 100
|
||||
}
|
||||
|
||||
// Requests search results for out-of-network replies to a user Id
|
||||
class OutOfNetworkRepliesToUserIdSearchResultsTransform(
|
||||
searchClient: SearchClient,
|
||||
statsReceiver: StatsReceiver,
|
||||
logSearchDebugInfo: Boolean = true)
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
private[this] val maxCountStat = statsReceiver.stat("maxCount")
|
||||
private[this] val numResultsFromSearchStat = statsReceiver.stat("numResultsFromSearch")
|
||||
private[this] val earlybirdScoreX100Stat = statsReceiver.stat("earlybirdScoreX100")
|
||||
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
val maxCount = envelope.query.maxCount
|
||||
.getOrElse(OutOfNetworkRepliesToUserIdSearchResultsTransform.DefaultMaxTweetCount)
|
||||
maxCountStat.add(maxCount)
|
||||
|
||||
envelope.followGraphData.followedUserIdsFuture
|
||||
.flatMap {
|
||||
case followedIds =>
|
||||
searchClient
|
||||
.getOutOfNetworkRepliesToUserId(
|
||||
userId = envelope.query.userId,
|
||||
followedUserIds = followedIds.toSet,
|
||||
maxCount = maxCount,
|
||||
earlybirdOptions = envelope.query.earlybirdOptions,
|
||||
logSearchDebugInfo
|
||||
).map { results =>
|
||||
numResultsFromSearchStat.add(results.size)
|
||||
results.foreach { result =>
|
||||
val earlybirdScoreX100 =
|
||||
result.metadata.flatMap(_.score).getOrElse(0.0).toFloat * 100
|
||||
earlybirdScoreX100Stat.add(earlybirdScoreX100)
|
||||
}
|
||||
envelope.copy(searchResults = results)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,31 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope
|
||||
import com.twitter.timelines.earlybird.common.utils.EarlybirdFeaturesHydrator
|
||||
import com.twitter.util.Future
|
||||
|
||||
object OutOfNetworkTweetsSearchFeaturesHydrationTransform
|
||||
extends FutureArrow[
|
||||
HydratedCandidatesAndFeaturesEnvelope,
|
||||
HydratedCandidatesAndFeaturesEnvelope
|
||||
] {
|
||||
override def apply(
|
||||
request: HydratedCandidatesAndFeaturesEnvelope
|
||||
): Future[HydratedCandidatesAndFeaturesEnvelope] = {
|
||||
val featuresByTweetId = EarlybirdFeaturesHydrator.hydrate(
|
||||
searcherUserId = request.candidateEnvelope.query.userId,
|
||||
searcherProfileInfo = request.userProfileInfo,
|
||||
followedUserIds = Seq.empty,
|
||||
mutuallyFollowingUserIds = Set.empty,
|
||||
userLanguages = request.userLanguages,
|
||||
uiLanguageCode = request.candidateEnvelope.query.deviceContext.flatMap(_.languageCode),
|
||||
searchResults = request.candidateEnvelope.searchResults,
|
||||
sourceTweetSearchResults = Seq.empty,
|
||||
tweets = request.candidateEnvelope.hydratedTweets.outerTweets,
|
||||
sourceTweets = Seq.empty
|
||||
)
|
||||
|
||||
Future.value(request.copy(features = featuresByTweetId))
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,29 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelines.clients.relevance_search.SearchClient
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.util.Future
|
||||
|
||||
trait RecapHydrationSearchResultsTransformBase
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
protected def statsReceiver: StatsReceiver
|
||||
protected def searchClient: SearchClient
|
||||
private[this] val numResultsFromSearchStat = statsReceiver.stat("numResultsFromSearch")
|
||||
|
||||
def tweetIdsToHydrate(envelope: CandidateEnvelope): Seq[TweetId]
|
||||
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
searchClient
|
||||
.getTweetsScoredForRecap(
|
||||
envelope.query.userId,
|
||||
tweetIdsToHydrate(envelope),
|
||||
envelope.query.earlybirdOptions
|
||||
).map { results =>
|
||||
numResultsFromSearchStat.add(results.size)
|
||||
envelope.copy(searchResults = results)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,89 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
|
||||
import com.twitter.timelineranker.model.TweetIdRange
|
||||
import com.twitter.timelineranker.parameters.recap.RecapParams
|
||||
import com.twitter.timelines.clients.relevance_search.SearchClient
|
||||
import com.twitter.timelines.clients.relevance_search.SearchClient.TweetTypes
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* Fetch recap/recycled search results using the search client
|
||||
* and populate them into the CandidateEnvelope
|
||||
*/
|
||||
class RecapSearchResultsTransform(
|
||||
searchClient: SearchClient,
|
||||
maxCountProvider: DependencyProvider[Int],
|
||||
returnAllResultsProvider: DependencyProvider[Boolean],
|
||||
relevanceOptionsMaxHitsToProcessProvider: DependencyProvider[Int],
|
||||
enableExcludeSourceTweetIdsProvider: DependencyProvider[Boolean],
|
||||
enableSettingTweetTypesWithTweetKindOptionProvider: DependencyProvider[Boolean],
|
||||
perRequestSearchClientIdProvider: DependencyProvider[Option[String]],
|
||||
relevanceSearchProvider: DependencyProvider[Boolean] =
|
||||
DependencyProvider.from(RecapParams.EnableRelevanceSearchParam),
|
||||
statsReceiver: StatsReceiver,
|
||||
logSearchDebugInfo: Boolean = true)
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
private[this] val maxCountStat = statsReceiver.stat("maxCount")
|
||||
private[this] val numResultsFromSearchStat = statsReceiver.stat("numResultsFromSearch")
|
||||
private[this] val excludedTweetIdsStat = statsReceiver.stat("excludedTweetIds")
|
||||
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
val maxCount = maxCountProvider(envelope.query)
|
||||
maxCountStat.add(maxCount)
|
||||
|
||||
val excludedTweetIdsOpt = envelope.query.excludedTweetIds
|
||||
excludedTweetIdsOpt.foreach { excludedTweetIds =>
|
||||
excludedTweetIdsStat.add(excludedTweetIds.size)
|
||||
}
|
||||
|
||||
val tweetIdRange = envelope.query.range
|
||||
.map(TweetIdRange.fromTimelineRange)
|
||||
.getOrElse(TweetIdRange.default)
|
||||
|
||||
val beforeTweetIdExclusive = tweetIdRange.toId
|
||||
val afterTweetIdExclusive = tweetIdRange.fromId
|
||||
|
||||
val returnAllResults = returnAllResultsProvider(envelope.query)
|
||||
val relevanceOptionsMaxHitsToProcess = relevanceOptionsMaxHitsToProcessProvider(envelope.query)
|
||||
|
||||
Future
|
||||
.join(
|
||||
envelope.followGraphData.followedUserIdsFuture,
|
||||
envelope.followGraphData.retweetsMutedUserIdsFuture
|
||||
).flatMap {
|
||||
case (followedIds, retweetsMutedIds) =>
|
||||
val followedIdsIncludingSelf = followedIds.toSet + envelope.query.userId
|
||||
|
||||
searchClient
|
||||
.getUsersTweetsForRecap(
|
||||
userId = envelope.query.userId,
|
||||
followedUserIds = followedIdsIncludingSelf,
|
||||
retweetsMutedUserIds = retweetsMutedIds,
|
||||
maxCount = maxCount,
|
||||
tweetTypes = TweetTypes.fromTweetKindOption(envelope.query.options),
|
||||
searchOperator = envelope.query.searchOperator,
|
||||
beforeTweetIdExclusive = beforeTweetIdExclusive,
|
||||
afterTweetIdExclusive = afterTweetIdExclusive,
|
||||
enableSettingTweetTypesWithTweetKindOption =
|
||||
enableSettingTweetTypesWithTweetKindOptionProvider(envelope.query),
|
||||
excludedTweetIds = excludedTweetIdsOpt,
|
||||
earlybirdOptions = envelope.query.earlybirdOptions,
|
||||
getOnlyProtectedTweets = false,
|
||||
logSearchDebugInfo = logSearchDebugInfo,
|
||||
returnAllResults = returnAllResults,
|
||||
enableExcludeSourceTweetIdsQuery =
|
||||
enableExcludeSourceTweetIdsProvider(envelope.query),
|
||||
relevanceSearch = relevanceSearchProvider(envelope.query),
|
||||
searchClientId = perRequestSearchClientIdProvider(envelope.query),
|
||||
relevanceOptionsMaxHitsToProcess = relevanceOptionsMaxHitsToProcess
|
||||
).map { results =>
|
||||
numResultsFromSearchStat.add(results.size)
|
||||
envelope.copy(searchResults = results)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,67 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
|
||||
import com.twitter.timelineranker.util.SearchResultUtil
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* Truncate the search results by score. Assumes that the search results are sorted in
|
||||
* score-descending order unless extraSortBeforeTruncation is set to true.
|
||||
*
|
||||
* This transform has two main use cases:
|
||||
*
|
||||
* - when returnAllResults is set to true, earlybird returns (numResultsPerShard * number of shards)
|
||||
* results. this transform is then used to further truncate the result, so that the size will be the
|
||||
* same as when returnAllResults is set to false.
|
||||
*
|
||||
* - we retrieve extra number of results from earlybird, as specified in MaxCountMultiplierParam,
|
||||
* so that we are left with sufficient number of candidates after hydration and filtering.
|
||||
* this transform will be used to get rid of extra results we ended up not using.
|
||||
*/
|
||||
class RecapSearchResultsTruncationTransform(
|
||||
extraSortBeforeTruncationGate: DependencyProvider[Boolean],
|
||||
maxCountProvider: DependencyProvider[Int],
|
||||
statsReceiver: StatsReceiver)
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
private[this] val postTruncationSizeStat = statsReceiver.stat("postTruncationSize")
|
||||
private[this] val earlybirdScoreX100Stat = statsReceiver.stat("earlybirdScoreX100")
|
||||
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
val sortBeforeTruncation = extraSortBeforeTruncationGate(envelope.query)
|
||||
val maxCount = maxCountProvider(envelope.query)
|
||||
val searchResults = envelope.searchResults
|
||||
|
||||
// set aside results that are marked by isRandomTweet field
|
||||
val (randomTweetSeq, searchResultsExcludingRandom) = searchResults.partition { result =>
|
||||
result.tweetFeatures.flatMap(_.isRandomTweet).getOrElse(false)
|
||||
}
|
||||
|
||||
// sort and truncate searchResults other than the random tweet
|
||||
val maxCountExcludingRandom = Math.max(0, maxCount - randomTweetSeq.size)
|
||||
|
||||
val truncatedResultsExcludingRandom =
|
||||
if (sortBeforeTruncation || searchResultsExcludingRandom.size > maxCountExcludingRandom) {
|
||||
val sorted = if (sortBeforeTruncation) {
|
||||
searchResultsExcludingRandom.sortWith(
|
||||
SearchResultUtil.getScore(_) > SearchResultUtil.getScore(_))
|
||||
} else searchResultsExcludingRandom
|
||||
sorted.take(maxCountExcludingRandom)
|
||||
} else searchResultsExcludingRandom
|
||||
|
||||
// put back the random tweet set aside previously
|
||||
val allTruncatedResults = truncatedResultsExcludingRandom ++ randomTweetSeq
|
||||
|
||||
// stats
|
||||
postTruncationSizeStat.add(allTruncatedResults.size)
|
||||
allTruncatedResults.foreach { result =>
|
||||
val earlybirdScoreX100 =
|
||||
result.metadata.flatMap(_.score).getOrElse(0.0).toFloat * 100
|
||||
earlybirdScoreX100Stat.add(earlybirdScoreX100)
|
||||
}
|
||||
|
||||
Future.value(envelope.copy(searchResults = allTruncatedResults))
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,23 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.util.Future
|
||||
import scala.collection.mutable
|
||||
|
||||
/**
|
||||
* Remove duplicate search results and order them reverse-chron.
|
||||
*/
|
||||
object SearchResultDedupAndSortingTransform
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
val seenTweetIds = mutable.Set.empty[TweetId]
|
||||
val dedupedResults = envelope.searchResults
|
||||
.filter(result => seenTweetIds.add(result.id))
|
||||
.sortBy(_.id)(Ordering[TweetId].reverse)
|
||||
|
||||
val transformedEnvelope = envelope.copy(searchResults = dedupedResults)
|
||||
Future.value(transformedEnvelope)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,62 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.search.earlybird.thriftscala.ThriftSearchResult
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
|
||||
import com.twitter.timelineranker.util.SourceTweetsUtil
|
||||
import com.twitter.timelines.clients.relevance_search.SearchClient
|
||||
import com.twitter.timelines.util.FailOpenHandler
|
||||
import com.twitter.util.Future
|
||||
|
||||
object SourceTweetsSearchResultsTransform {
|
||||
val EmptySearchResults: Seq[ThriftSearchResult] = Seq.empty[ThriftSearchResult]
|
||||
val EmptySearchResultsFuture: Future[Seq[ThriftSearchResult]] = Future.value(EmptySearchResults)
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch source tweets for a given set of search results
|
||||
* Collects ids of source tweets, including extended reply and reply source tweets if needed,
|
||||
* fetches those tweets from search and populates them into the envelope
|
||||
*/
|
||||
class SourceTweetsSearchResultsTransform(
|
||||
searchClient: SearchClient,
|
||||
failOpenHandler: FailOpenHandler,
|
||||
hydrateReplyRootTweetProvider: DependencyProvider[Boolean],
|
||||
perRequestSourceSearchClientIdProvider: DependencyProvider[Option[String]],
|
||||
statsReceiver: StatsReceiver)
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
import SourceTweetsSearchResultsTransform._
|
||||
|
||||
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
|
||||
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
failOpenHandler {
|
||||
envelope.followGraphData.followedUserIdsFuture.flatMap { followedUserIds =>
|
||||
// NOTE: tweetIds are pre-computed as a performance optimisation.
|
||||
val searchResultsTweetIds = envelope.searchResults.map(_.id).toSet
|
||||
val sourceTweetIds = SourceTweetsUtil.getSourceTweetIds(
|
||||
searchResults = envelope.searchResults,
|
||||
searchResultsTweetIds = searchResultsTweetIds,
|
||||
followedUserIds = followedUserIds,
|
||||
shouldIncludeReplyRootTweets = hydrateReplyRootTweetProvider(envelope.query),
|
||||
statsReceiver = scopedStatsReceiver
|
||||
)
|
||||
if (sourceTweetIds.isEmpty) {
|
||||
EmptySearchResultsFuture
|
||||
} else {
|
||||
searchClient.getTweetsScoredForRecap(
|
||||
userId = envelope.query.userId,
|
||||
tweetIds = sourceTweetIds,
|
||||
earlybirdOptions = envelope.query.earlybirdOptions,
|
||||
logSearchDebugInfo = false,
|
||||
searchClientId = perRequestSourceSearchClientIdProvider(envelope.query)
|
||||
)
|
||||
}
|
||||
}
|
||||
} { _: Throwable => EmptySearchResultsFuture }.map { sourceSearchResults =>
|
||||
envelope.copy(sourceSearchResults = sourceSearchResults)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,37 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.search.earlybird.thriftscala.ThriftSearchResult
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelines.model.tweet.HydratedTweet
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* trims searchResults to match with hydratedTweets
|
||||
* (if we previously filtered out hydrated tweets, this transform filters the search result set
|
||||
* down to match the hydrated tweets.)
|
||||
*/
|
||||
object TrimToMatchHydratedTweetsTransform
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
val filteredSearchResults =
|
||||
trimSearchResults(envelope.searchResults, envelope.hydratedTweets.outerTweets)
|
||||
val filteredSourceSearchResults =
|
||||
trimSearchResults(envelope.sourceSearchResults, envelope.sourceHydratedTweets.outerTweets)
|
||||
|
||||
Future.value(
|
||||
envelope.copy(
|
||||
searchResults = filteredSearchResults,
|
||||
sourceSearchResults = filteredSourceSearchResults
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
private def trimSearchResults(
|
||||
searchResults: Seq[ThriftSearchResult],
|
||||
hydratedTweets: Seq[HydratedTweet]
|
||||
): Seq[ThriftSearchResult] = {
|
||||
val filteredTweetIds = hydratedTweets.map(_.tweetId).toSet
|
||||
searchResults.filter(result => filteredTweetIds.contains(result.id))
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,57 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
|
||||
import com.twitter.timelineranker.util.SourceTweetsUtil
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* trims elements of the envelope other than the searchResults
|
||||
* (i.e. sourceSearchResults, hydratedTweets, sourceHydratedTweets) to match with searchResults.
|
||||
*/
|
||||
class TrimToMatchSearchResultsTransform(
|
||||
hydrateReplyRootTweetProvider: DependencyProvider[Boolean],
|
||||
statsReceiver: StatsReceiver)
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
|
||||
private val scopedStatsReceiver = statsReceiver.scope(getClass.getSimpleName)
|
||||
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
val searchResults = envelope.searchResults
|
||||
val searchResultsIds = searchResults.map(_.id).toSet
|
||||
|
||||
// Trim rest of the seqs to match top search results.
|
||||
val hydratedTweets = envelope.hydratedTweets.outerTweets
|
||||
val topHydratedTweets = hydratedTweets.filter(ht => searchResultsIds.contains(ht.tweetId))
|
||||
|
||||
envelope.followGraphData.followedUserIdsFuture.map { followedUserIds =>
|
||||
val sourceTweetIdsOfTopResults =
|
||||
SourceTweetsUtil
|
||||
.getSourceTweetIds(
|
||||
searchResults = searchResults,
|
||||
searchResultsTweetIds = searchResultsIds,
|
||||
followedUserIds = followedUserIds,
|
||||
shouldIncludeReplyRootTweets = hydrateReplyRootTweetProvider(envelope.query),
|
||||
statsReceiver = scopedStatsReceiver
|
||||
).toSet
|
||||
val sourceTweetSearchResultsForTopN =
|
||||
envelope.sourceSearchResults.filter(r => sourceTweetIdsOfTopResults.contains(r.id))
|
||||
val hydratedSourceTweetsForTopN =
|
||||
envelope.sourceHydratedTweets.outerTweets.filter(ht =>
|
||||
sourceTweetIdsOfTopResults.contains(ht.tweetId))
|
||||
|
||||
val hydratedTweetsForEnvelope = envelope.hydratedTweets.copy(outerTweets = topHydratedTweets)
|
||||
val hydratedSourceTweetsForEnvelope =
|
||||
envelope.sourceHydratedTweets.copy(outerTweets = hydratedSourceTweetsForTopN)
|
||||
|
||||
envelope.copy(
|
||||
hydratedTweets = hydratedTweetsForEnvelope,
|
||||
searchResults = searchResults,
|
||||
sourceHydratedTweets = hydratedSourceTweetsForEnvelope,
|
||||
sourceSearchResults = sourceTweetSearchResultsForTopN
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,62 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.finagle.IndividualRequestTimeoutException
|
||||
import com.twitter.search.earlybird.thriftscala.ThriftSearchResult
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.core.HydratedTweets
|
||||
import com.twitter.timelineranker.model.PartiallyHydratedTweet
|
||||
import com.twitter.timelines.model.tweet.HydratedTweet
|
||||
import com.twitter.util.Future
|
||||
|
||||
object TweetHydrationTransform {
|
||||
val EmptyHydratedTweets: HydratedTweets =
|
||||
HydratedTweets(Seq.empty[HydratedTweet], Seq.empty[HydratedTweet])
|
||||
val EmptyHydratedTweetsFuture: Future[HydratedTweets] = Future.value(EmptyHydratedTweets)
|
||||
}
|
||||
|
||||
object CandidateTweetHydrationTransform extends TweetHydrationTransform {
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
hydrate(
|
||||
searchResults = envelope.searchResults,
|
||||
envelope = envelope
|
||||
).map { tweets => envelope.copy(hydratedTweets = tweets) }
|
||||
}
|
||||
}
|
||||
|
||||
object SourceTweetHydrationTransform extends TweetHydrationTransform {
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
hydrate(
|
||||
searchResults = envelope.sourceSearchResults,
|
||||
envelope = envelope
|
||||
).map { tweets => envelope.copy(sourceHydratedTweets = tweets) }
|
||||
}
|
||||
}
|
||||
|
||||
// Static IRTE to indicate timeout in tweet hydrator. Placeholder timeout duration of 0 millis is used
|
||||
// since we are only concerned with the source of the exception.
|
||||
object TweetHydrationTimeoutException extends IndividualRequestTimeoutException(0.millis) {
|
||||
serviceName = "tweetHydrator"
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform which hydrates tweets in the CandidateEnvelope
|
||||
**/
|
||||
trait TweetHydrationTransform extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
|
||||
import TweetHydrationTransform._
|
||||
|
||||
protected def hydrate(
|
||||
searchResults: Seq[ThriftSearchResult],
|
||||
envelope: CandidateEnvelope
|
||||
): Future[HydratedTweets] = {
|
||||
if (searchResults.nonEmpty) {
|
||||
Future.value(
|
||||
HydratedTweets(searchResults.map(PartiallyHydratedTweet.fromSearchResult))
|
||||
)
|
||||
} else {
|
||||
EmptyHydratedTweetsFuture
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,85 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.servo.util.Gate
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery
|
||||
import com.twitter.timelineranker.parameters.recap.RecapParams
|
||||
import com.twitter.timelineranker.parameters.uteg_liked_by_tweets.UtegLikedByTweetsParams
|
||||
import com.twitter.timelineranker.util.TweetFilters
|
||||
import com.twitter.timelines.common.model.TweetKindOption
|
||||
import com.twitter.util.Future
|
||||
import scala.collection.mutable
|
||||
|
||||
object TweetKindOptionHydratedTweetsFilterTransform {
|
||||
private[common] val enableExpandedExtendedRepliesGate: Gate[RecapQuery] =
|
||||
RecapQuery.paramGate(RecapParams.EnableExpandedExtendedRepliesFilterParam)
|
||||
|
||||
private[common] val excludeRecommendedRepliesToNonFollowedUsersGate: Gate[RecapQuery] =
|
||||
RecapQuery.paramGate(
|
||||
UtegLikedByTweetsParams.UTEGRecommendationsFilter.ExcludeRecommendedRepliesToNonFollowedUsersParam)
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter hydrated tweets dynamically based on TweetKindOptions in the query.
|
||||
*/
|
||||
class TweetKindOptionHydratedTweetsFilterTransform(
|
||||
useFollowGraphData: Boolean,
|
||||
useSourceTweets: Boolean,
|
||||
statsReceiver: StatsReceiver)
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
import TweetKindOptionHydratedTweetsFilterTransform._
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
val filters = convertToFilters(envelope)
|
||||
|
||||
val filterTransform = if (filters == TweetFilters.ValueSet.empty) {
|
||||
FutureArrow.identity[CandidateEnvelope]
|
||||
} else {
|
||||
new HydratedTweetsFilterTransform(
|
||||
outerFilters = filters,
|
||||
innerFilters = TweetFilters.None,
|
||||
useFollowGraphData = useFollowGraphData,
|
||||
useSourceTweets = useSourceTweets,
|
||||
statsReceiver = statsReceiver,
|
||||
numRetweetsAllowed = HydratedTweetsFilterTransform.NumDuplicateRetweetsAllowed
|
||||
)
|
||||
}
|
||||
|
||||
filterTransform.apply(envelope)
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts the given query options to equivalent TweetFilter values.
|
||||
*
|
||||
* Note:
|
||||
* -- The semantic of TweetKindOption is opposite of that of TweetFilters.
|
||||
* TweetKindOption values are of the form IncludeX. That is, they result in X being added.
|
||||
* TweetFilters values specify what to exclude.
|
||||
* -- IncludeExtendedReplies requires IncludeReplies to be also specified to be effective.
|
||||
*/
|
||||
private[common] def convertToFilters(envelope: CandidateEnvelope): TweetFilters.ValueSet = {
|
||||
val queryOptions = envelope.query.options
|
||||
val filters = mutable.Set.empty[TweetFilters.Value]
|
||||
if (queryOptions.contains(TweetKindOption.IncludeReplies)) {
|
||||
if (excludeRecommendedRepliesToNonFollowedUsersGate(
|
||||
envelope.query) && envelope.query.utegLikedByTweetsOptions.isDefined) {
|
||||
filters += TweetFilters.RecommendedRepliesToNotFollowedUsers
|
||||
} else if (queryOptions.contains(TweetKindOption.IncludeExtendedReplies)) {
|
||||
if (enableExpandedExtendedRepliesGate(envelope.query)) {
|
||||
filters += TweetFilters.NotValidExpandedExtendedReplies
|
||||
} else {
|
||||
filters += TweetFilters.NotQualifiedExtendedReplies
|
||||
}
|
||||
} else {
|
||||
filters += TweetFilters.ExtendedReplies
|
||||
}
|
||||
} else {
|
||||
filters += TweetFilters.Replies
|
||||
}
|
||||
if (!queryOptions.contains(TweetKindOption.IncludeRetweets)) {
|
||||
filters += TweetFilters.Retweets
|
||||
}
|
||||
TweetFilters.ValueSet.empty ++ filters
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,30 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.search.common.constants.thriftscala.ThriftLanguage
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.model.RecapQuery
|
||||
import com.twitter.timelines.clients.manhattan.LanguageUtils
|
||||
import com.twitter.timelines.clients.manhattan.UserMetadataClient
|
||||
import com.twitter.timelines.util.FailOpenHandler
|
||||
import com.twitter.util.Future
|
||||
import com.twitter.service.metastore.gen.thriftscala.UserLanguages
|
||||
|
||||
object UserLanguagesTransform {
|
||||
val EmptyUserLanguagesFuture: Future[UserLanguages] =
|
||||
Future.value(UserMetadataClient.EmptyUserLanguages)
|
||||
}
|
||||
|
||||
/**
|
||||
* FutureArrow which fetches user languages
|
||||
* It should be run in parallel with the main pipeline which fetches and hydrates CandidateTweets
|
||||
*/
|
||||
class UserLanguagesTransform(handler: FailOpenHandler, userMetadataClient: UserMetadataClient)
|
||||
extends FutureArrow[RecapQuery, Seq[ThriftLanguage]] {
|
||||
override def apply(request: RecapQuery): Future[Seq[ThriftLanguage]] = {
|
||||
import UserLanguagesTransform._
|
||||
|
||||
handler {
|
||||
userMetadataClient.getUserLanguages(request.userId)
|
||||
} { _: Throwable => EmptyUserLanguagesFuture }
|
||||
}.map(LanguageUtils.computeLanguages(_))
|
||||
}
|
||||
@ -0,0 +1,29 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.model.RecapQuery
|
||||
import com.twitter.timelines.clients.gizmoduck.GizmoduckClient
|
||||
import com.twitter.timelines.clients.gizmoduck.UserProfileInfo
|
||||
import com.twitter.timelines.util.FailOpenHandler
|
||||
import com.twitter.util.Future
|
||||
|
||||
object UserProfileInfoTransform {
|
||||
val EmptyUserProfileInfo: UserProfileInfo = UserProfileInfo(None, None, None, None)
|
||||
val EmptyUserProfileInfoFuture: Future[UserProfileInfo] = Future.value(EmptyUserProfileInfo)
|
||||
}
|
||||
|
||||
/**
|
||||
* FutureArrow which fetches user profile info
|
||||
* It should be run in parallel with the main pipeline which fetches and hydrates CandidateTweets
|
||||
*/
|
||||
class UserProfileInfoTransform(handler: FailOpenHandler, gizmoduckClient: GizmoduckClient)
|
||||
extends FutureArrow[RecapQuery, UserProfileInfo] {
|
||||
import UserProfileInfoTransform._
|
||||
override def apply(request: RecapQuery): Future[UserProfileInfo] = {
|
||||
handler {
|
||||
gizmoduckClient.getProfileInfo(request.userId).map { profileInfoOpt =>
|
||||
profileInfoOpt.getOrElse(EmptyUserProfileInfo)
|
||||
}
|
||||
} { _: Throwable => EmptyUserProfileInfoFuture }
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,22 @@
|
||||
package com.twitter.timelineranker.common
|
||||
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.core.HydratedTweets
|
||||
import com.twitter.timelines.visibility.VisibilityEnforcer
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* Transform which uses an instance of a VisiblityEnforcer to filter down HydratedTweets
|
||||
*/
|
||||
class VisibilityEnforcingTransform(visibilityEnforcer: VisibilityEnforcer)
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
visibilityEnforcer.apply(Some(envelope.query.userId), envelope.hydratedTweets.outerTweets).map {
|
||||
visibleTweets =>
|
||||
val innerTweets = envelope.hydratedTweets.innerTweets
|
||||
envelope.copy(
|
||||
hydratedTweets = HydratedTweets(outerTweets = visibleTweets, innerTweets = innerTweets))
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,65 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/twitter/storehaus:core",
|
||||
"3rdparty/jvm/org/apache/thrift:libthrift",
|
||||
"abdecider",
|
||||
"cortex-tweet-annotate/service/src/main/thrift:thrift-scala",
|
||||
"decider",
|
||||
"featureswitches/featureswitches-core/src/main/scala",
|
||||
"finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication",
|
||||
"finagle/finagle-core/src/main",
|
||||
"finagle/finagle-memcached/src/main/scala",
|
||||
"finagle/finagle-stats",
|
||||
"finagle/finagle-thrift/src/main/java",
|
||||
"finagle/finagle-thrift/src/main/scala",
|
||||
"finagle/finagle-thriftmux/src/main/scala",
|
||||
"loglens/loglens-logging/src/main/scala",
|
||||
"merlin/util/src/main/scala",
|
||||
"servo/decider",
|
||||
"servo/request/src/main/scala",
|
||||
"src/thrift/com/twitter/gizmoduck:thrift-scala",
|
||||
"src/thrift/com/twitter/manhattan:v1-scala",
|
||||
"src/thrift/com/twitter/merlin:thrift-scala",
|
||||
"src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
|
||||
"src/thrift/com/twitter/search:earlybird-scala",
|
||||
"src/thrift/com/twitter/socialgraph:thrift-scala",
|
||||
"src/thrift/com/twitter/timelineranker:thrift-scala",
|
||||
"src/thrift/com/twitter/timelineservice:thrift-scala",
|
||||
"src/thrift/com/twitter/tweetypie:service-scala",
|
||||
"src/thrift/com/twitter/tweetypie:tweet-scala",
|
||||
"strato/src/main/scala/com/twitter/strato/client",
|
||||
"timelineranker/server/config",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/clients",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/clients/content_features_cache",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/decider",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model",
|
||||
"timelines:authorization",
|
||||
"timelines:config",
|
||||
"timelines:features",
|
||||
"timelines:util",
|
||||
"timelines:visibility",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/manhattan",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/memcache_common",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/socialgraph",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/strato/realgraph",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/tweetypie",
|
||||
"timelines/src/main/scala/com/twitter/timelines/config/configapi",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util/logging",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util/stats",
|
||||
"timelineservice/common/src/main/scala/com/twitter/timelineservice/model",
|
||||
"twitter-server-internal",
|
||||
"util/util-app",
|
||||
"util/util-core:util-core-util",
|
||||
"util/util-core/src/main/scala/com/twitter/conversions",
|
||||
"util/util-stats/src/main/scala",
|
||||
],
|
||||
exports = [
|
||||
"src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,119 @@
|
||||
package com.twitter.timelineranker.config
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.util.Duration
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
/**
|
||||
* Information about a single method call.
|
||||
*
|
||||
* The purpose of this class is to allow one to express a call graph and latency associated with each (sub)call.
|
||||
* Once a call graph is defined, calling getOverAllLatency() off the top level call returns total time taken by that call.
|
||||
* That value can then be compared with the timeout budget allocated to that call to see if the
|
||||
* value fits within the overall timeout budget of that call.
|
||||
*
|
||||
* This is useful in case of a complex call graph where it is hard to mentally estimate the effect on
|
||||
* overall latency when updating timeout value of one or more sub-calls.
|
||||
*
|
||||
* @param methodName name of the called method.
|
||||
* @param latency P999 Latency incurred in calling a service if the method calls an external service. Otherwise 0.
|
||||
* @param dependsOn Other calls that this call depends on.
|
||||
*/
|
||||
case class Call(
|
||||
methodName: String,
|
||||
latency: Duration = 0.milliseconds,
|
||||
dependsOn: Seq[Call] = Nil) {
|
||||
|
||||
/**
|
||||
* Latency incurred in this call as well as recursively all calls this call depends on.
|
||||
*/
|
||||
def getOverAllLatency: Duration = {
|
||||
val dependencyLatency = if (dependsOn.isEmpty) {
|
||||
0.milliseconds
|
||||
} else {
|
||||
dependsOn.map(_.getOverAllLatency).max
|
||||
}
|
||||
latency + dependencyLatency
|
||||
}
|
||||
|
||||
/**
|
||||
* Call paths starting at this call and recursively traversing all dependencies.
|
||||
* Typically used for debugging or logging.
|
||||
*/
|
||||
def getLatencyPaths: String = {
|
||||
val sb = new StringBuilder
|
||||
getLatencyPaths(sb, 1)
|
||||
sb.toString
|
||||
}
|
||||
|
||||
def getLatencyPaths(sb: StringBuilder, level: Int): Unit = {
|
||||
sb.append(s"${getPrefix(level)} ${getLatencyString(getOverAllLatency)} $methodName\n")
|
||||
if ((latency > 0.milliseconds) && !dependsOn.isEmpty) {
|
||||
sb.append(s"${getPrefix(level + 1)} ${getLatencyString(latency)} self\n")
|
||||
}
|
||||
dependsOn.foreach(_.getLatencyPaths(sb, level + 1))
|
||||
}
|
||||
|
||||
private def getLatencyString(latencyValue: Duration): String = {
|
||||
val latencyMs = latencyValue.inUnit(TimeUnit.MILLISECONDS)
|
||||
f"[$latencyMs%3d]"
|
||||
}
|
||||
|
||||
private def getPrefix(level: Int): String = {
|
||||
" " * (level * 4) + "--"
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Information about the getRecapTweetCandidates call.
|
||||
*
|
||||
* Acronyms used:
|
||||
* : call internal to TLR
|
||||
* EB : Earlybird (search super root)
|
||||
* GZ : Gizmoduck
|
||||
* MH : Manhattan
|
||||
* SGS : Social graph service
|
||||
*
|
||||
* The latency values are based on p9999 values observed over 1 week.
|
||||
*/
|
||||
object GetRecycledTweetCandidatesCall {
|
||||
val getUserProfileInfo: Call = Call("GZ.getUserProfileInfo", 200.milliseconds)
|
||||
val getUserLanguages: Call = Call("MH.getUserLanguages", 300.milliseconds) // p99: 15ms
|
||||
|
||||
val getFollowing: Call = Call("SGS.getFollowing", 250.milliseconds) // p99: 75ms
|
||||
val getMutuallyFollowing: Call =
|
||||
Call("SGS.getMutuallyFollowing", 400.milliseconds, Seq(getFollowing)) // p99: 100
|
||||
val getVisibilityProfiles: Call =
|
||||
Call("SGS.getVisibilityProfiles", 400.milliseconds, Seq(getFollowing)) // p99: 100
|
||||
val getVisibilityData: Call = Call(
|
||||
"getVisibilityData",
|
||||
dependsOn = Seq(getFollowing, getMutuallyFollowing, getVisibilityProfiles)
|
||||
)
|
||||
val getTweetsForRecapRegular: Call =
|
||||
Call("EB.getTweetsForRecap(regular)", 500.milliseconds, Seq(getVisibilityData)) // p99: 250
|
||||
val getTweetsForRecapProtected: Call =
|
||||
Call("EB.getTweetsForRecap(protected)", 250.milliseconds, Seq(getVisibilityData)) // p99: 150
|
||||
val getSearchResults: Call =
|
||||
Call("getSearchResults", dependsOn = Seq(getTweetsForRecapRegular, getTweetsForRecapProtected))
|
||||
val getTweetsScoredForRecap: Call =
|
||||
Call("EB.getTweetsScoredForRecap", 400.milliseconds, Seq(getSearchResults)) // p99: 100
|
||||
|
||||
val hydrateSearchResults: Call = Call("hydrateSearchResults")
|
||||
val getSourceTweetSearchResults: Call =
|
||||
Call("getSourceTweetSearchResults", dependsOn = Seq(getSearchResults))
|
||||
val hydrateTweets: Call =
|
||||
Call("hydrateTweets", dependsOn = Seq(getSearchResults, hydrateSearchResults))
|
||||
val hydrateSourceTweets: Call =
|
||||
Call("hydrateSourceTweets", dependsOn = Seq(getSourceTweetSearchResults, hydrateSearchResults))
|
||||
val topLevel: Call = Call(
|
||||
"getRecapTweetCandidates",
|
||||
dependsOn = Seq(
|
||||
getUserProfileInfo,
|
||||
getUserLanguages,
|
||||
getVisibilityData,
|
||||
getSearchResults,
|
||||
hydrateSearchResults,
|
||||
hydrateSourceTweets
|
||||
)
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,287 @@
|
||||
package com.twitter.timelineranker.config
|
||||
|
||||
import com.twitter.timelineranker.decider.DeciderKey._
|
||||
import com.twitter.timelines.authorization.TrustedPermission
|
||||
import com.twitter.timelines.authorization.RateLimitingTrustedPermission
|
||||
import com.twitter.timelines.authorization.RateLimitingUntrustedPermission
|
||||
import com.twitter.timelines.authorization.ClientDetails
|
||||
|
||||
object ClientAccessPermissions {
|
||||
// We want timelineranker locked down for requests outside of what's defined here.
|
||||
val DefaultRateLimit = 0d
|
||||
|
||||
def unknown(name: String): ClientDetails = {
|
||||
ClientDetails(name, RateLimitingUntrustedPermission(RateLimitOverrideUnknown, DefaultRateLimit))
|
||||
}
|
||||
|
||||
val All: Seq[ClientDetails] = Seq(
|
||||
/**
|
||||
* Production clients for timelinemixer.
|
||||
*/
|
||||
new ClientDetails(
|
||||
"timelinemixer.recap.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerRecapProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.recycled.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerRecycledProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.hydrate.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerHydrateProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.hydrate_recos.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerHydrateRecosProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.seed_author_ids.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerSeedAuthorsProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.simcluster.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerSimclusterProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.entity_tweets.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerEntityTweetsProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
/**
|
||||
* This client is whitelisted for timelinemixer only as it used by
|
||||
* List injection service which will not be migrated to timelinescorer.
|
||||
*/
|
||||
new ClientDetails(
|
||||
"timelinemixer.list.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerListProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.list_tweet.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerListTweetProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.community.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerCommunityProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.community_tweet.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerCommunityTweetProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.uteg_liked_by_tweets.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerUtegLikedByTweetsProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
/**
|
||||
* Production clients for timelinescorer. Most of these clients have their
|
||||
* equivalents under the timelinemixer scope (with exception of list injection
|
||||
* client).
|
||||
*/
|
||||
new ClientDetails(
|
||||
"timelinescorer.recap.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerRecapProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.recycled.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerRecycledProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.hydrate.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerHydrateProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.hydrate_recos.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerHydrateRecosProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.seed_author_ids.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerSeedAuthorsProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.simcluster.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerSimclusterProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.entity_tweets.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerEntityTweetsProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.list_tweet.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerListTweetProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.uteg_liked_by_tweets.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerUtegLikedByTweetsProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelineservice.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineServiceProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.hydrate_tweet_scoring.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineScorerHydrateTweetScoringProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.community_tweet.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerCommunityTweetProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.recommended_trend_tweet.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineScorerRecommendedTrendTweetProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.rec_topic_tweets.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineScorerRecTopicTweetsProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.popular_topic_tweets.prod",
|
||||
RateLimitingTrustedPermission(AllowTimelineScorerPopularTopicTweetsProd),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
/**
|
||||
* TimelineRanker utilities. Traffic proxy, warmups, and console.
|
||||
*/
|
||||
new ClientDetails(
|
||||
"timelineranker.proxy",
|
||||
RateLimitingTrustedPermission(AllowTimelineRankerProxy),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
TimelineRankerConstants.WarmupClientName,
|
||||
RateLimitingTrustedPermission(AllowTimelineRankerWarmup),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
TimelineRankerConstants.ForwardedClientName,
|
||||
RateLimitingTrustedPermission(AllowTimelineRankerWarmup),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelineranker.console",
|
||||
RateLimitingUntrustedPermission(RateLimitOverrideUnknown, 1d),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
/**
|
||||
* Staging clients.
|
||||
*/
|
||||
new ClientDetails(
|
||||
"timelinemixer.recap.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.recycled.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.hydrate.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.hydrate_recos.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.seed_author_ids.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.simcluster.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.entity_tweets.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.list.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.list_tweet.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.community.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.community_tweet.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.community_tweet.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.recommended_trend_tweet.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.uteg_liked_by_tweets.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinemixer.entity_tweets.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.hydrate_tweet_scoring.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.rec_topic_tweets.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelinescorer.popular_topic_tweets.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineMixerStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
),
|
||||
new ClientDetails(
|
||||
"timelineservice.staging",
|
||||
RateLimitingTrustedPermission(AllowTimelineServiceStaging),
|
||||
protectedWriteAccess = TrustedPermission
|
||||
)
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,86 @@
|
||||
package com.twitter.timelineranker.config
|
||||
|
||||
import com.twitter.servo.util.Gate
|
||||
import com.twitter.timelineranker.clients.ScopedCortexTweetQueryServiceClientFactory
|
||||
import com.twitter.timelines.clients.gizmoduck.ScopedGizmoduckClientFactory
|
||||
import com.twitter.timelines.clients.manhattan.ScopedUserMetadataClientFactory
|
||||
import com.twitter.timelines.clients.socialgraph.ScopedSocialGraphClientFactory
|
||||
import com.twitter.timelines.clients.strato.realgraph.ScopedRealGraphClientFactory
|
||||
import com.twitter.timelines.clients.tweetypie.AdditionalFieldConfig
|
||||
import com.twitter.timelines.clients.tweetypie.ScopedTweetyPieClientFactory
|
||||
import com.twitter.timelines.visibility.VisibilityEnforcerFactory
|
||||
import com.twitter.timelines.visibility.VisibilityProfileHydratorFactory
|
||||
import com.twitter.tweetypie.thriftscala.{Tweet => TTweet}
|
||||
|
||||
class ClientWrapperFactories(config: RuntimeConfiguration) {
|
||||
private[this] val statsReceiver = config.statsReceiver
|
||||
|
||||
val cortexTweetQueryServiceClientFactory: ScopedCortexTweetQueryServiceClientFactory =
|
||||
new ScopedCortexTweetQueryServiceClientFactory(
|
||||
config.underlyingClients.cortexTweetQueryServiceClient,
|
||||
statsReceiver = statsReceiver
|
||||
)
|
||||
|
||||
val gizmoduckClientFactory: ScopedGizmoduckClientFactory = new ScopedGizmoduckClientFactory(
|
||||
config.underlyingClients.gizmoduckClient,
|
||||
statsReceiver = statsReceiver
|
||||
)
|
||||
|
||||
val socialGraphClientFactory: ScopedSocialGraphClientFactory = new ScopedSocialGraphClientFactory(
|
||||
config.underlyingClients.sgsClient,
|
||||
statsReceiver
|
||||
)
|
||||
|
||||
val visibilityEnforcerFactory: VisibilityEnforcerFactory = new VisibilityEnforcerFactory(
|
||||
gizmoduckClientFactory,
|
||||
socialGraphClientFactory,
|
||||
statsReceiver
|
||||
)
|
||||
|
||||
val tweetyPieAdditionalFieldsToDisable: Seq[Short] = Seq(
|
||||
TTweet.MediaTagsField.id,
|
||||
TTweet.SchedulingInfoField.id,
|
||||
TTweet.EscherbirdEntityAnnotationsField.id,
|
||||
TTweet.CardReferenceField.id,
|
||||
TTweet.SelfPermalinkField.id,
|
||||
TTweet.ExtendedTweetMetadataField.id,
|
||||
TTweet.CommunitiesField.id,
|
||||
TTweet.VisibleTextRangeField.id
|
||||
)
|
||||
|
||||
val tweetyPieHighQoSClientFactory: ScopedTweetyPieClientFactory =
|
||||
new ScopedTweetyPieClientFactory(
|
||||
tweetyPieClient = config.underlyingClients.tweetyPieHighQoSClient,
|
||||
additionalFieldConfig = AdditionalFieldConfig(
|
||||
fieldDisablingGates = tweetyPieAdditionalFieldsToDisable.map(_ -> Gate.False).toMap
|
||||
),
|
||||
includePartialResults = Gate.False,
|
||||
statsReceiver = statsReceiver
|
||||
)
|
||||
|
||||
val tweetyPieLowQoSClientFactory: ScopedTweetyPieClientFactory = new ScopedTweetyPieClientFactory(
|
||||
tweetyPieClient = config.underlyingClients.tweetyPieLowQoSClient,
|
||||
additionalFieldConfig = AdditionalFieldConfig(
|
||||
fieldDisablingGates = tweetyPieAdditionalFieldsToDisable.map(_ -> Gate.False).toMap
|
||||
),
|
||||
includePartialResults = Gate.False,
|
||||
statsReceiver = statsReceiver
|
||||
)
|
||||
|
||||
val userMetadataClientFactory: ScopedUserMetadataClientFactory =
|
||||
new ScopedUserMetadataClientFactory(
|
||||
config.underlyingClients.manhattanStarbuckClient,
|
||||
TimelineRankerConstants.ManhattanStarbuckAppId,
|
||||
statsReceiver
|
||||
)
|
||||
|
||||
val visibilityProfileHydratorFactory: VisibilityProfileHydratorFactory =
|
||||
new VisibilityProfileHydratorFactory(
|
||||
gizmoduckClientFactory,
|
||||
socialGraphClientFactory,
|
||||
statsReceiver
|
||||
)
|
||||
|
||||
val realGraphClientFactory =
|
||||
new ScopedRealGraphClientFactory(config.underlyingClients.stratoClient, statsReceiver)
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
package com.twitter.timelineranker.config
|
||||
|
||||
import com.twitter.storehaus.Store
|
||||
import com.twitter.timelineranker.recap.model.ContentFeatures
|
||||
import com.twitter.timelines.model.TweetId
|
||||
class ClientWrappers(config: RuntimeConfiguration) {
|
||||
private[this] val backendClientConfiguration = config.underlyingClients
|
||||
|
||||
val contentFeaturesCache: Store[TweetId, ContentFeatures] =
|
||||
backendClientConfiguration.contentFeaturesCache
|
||||
}
|
||||
@ -0,0 +1,158 @@
|
||||
package com.twitter.timelineranker.config
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.conversions.PercentOps._
|
||||
import com.twitter.cortex_tweet_annotate.thriftscala.CortexTweetQueryService
|
||||
import com.twitter.finagle.ssl.OpportunisticTls
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finagle.thrift.ClientId
|
||||
import com.twitter.finagle.util.DefaultTimer
|
||||
import com.twitter.gizmoduck.thriftscala.{UserService => Gizmoduck}
|
||||
import com.twitter.manhattan.v1.thriftscala.{ManhattanCoordinator => ManhattanV1}
|
||||
import com.twitter.merlin.thriftscala.UserRolesService
|
||||
import com.twitter.recos.user_tweet_entity_graph.thriftscala.UserTweetEntityGraph
|
||||
import com.twitter.socialgraph.thriftscala.SocialGraphService
|
||||
import com.twitter.storehaus.Store
|
||||
import com.twitter.strato.client.Strato
|
||||
import com.twitter.strato.client.{Client => StratoClient}
|
||||
import com.twitter.timelineranker.clients.content_features_cache.ContentFeaturesMemcacheBuilder
|
||||
import com.twitter.timelineranker.recap.model.ContentFeatures
|
||||
import com.twitter.timelineranker.thriftscala.TimelineRanker
|
||||
import com.twitter.timelines.clients.memcache_common.StorehausMemcacheConfig
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.timelineservice.thriftscala.TimelineService
|
||||
import com.twitter.tweetypie.thriftscala.{TweetService => TweetyPie}
|
||||
import com.twitter.util.Timer
|
||||
import org.apache.thrift.protocol.TCompactProtocol
|
||||
|
||||
class DefaultUnderlyingClientConfiguration(flags: TimelineRankerFlags, statsReceiver: StatsReceiver)
|
||||
extends UnderlyingClientConfiguration(flags, statsReceiver) { top =>
|
||||
|
||||
val timer: Timer = DefaultTimer
|
||||
val twCachePrefix = "/srv#/prod/local/cache"
|
||||
|
||||
override val cortexTweetQueryServiceClient: CortexTweetQueryService.MethodPerEndpoint = {
|
||||
methodPerEndpointClient[
|
||||
CortexTweetQueryService.ServicePerEndpoint,
|
||||
CortexTweetQueryService.MethodPerEndpoint](
|
||||
thriftMuxClientBuilder("cortex-tweet-query", requireMutualTls = true)
|
||||
.dest("/s/cortex-tweet-annotate/cortex-tweet-query")
|
||||
.requestTimeout(200.milliseconds)
|
||||
.timeout(500.milliseconds)
|
||||
)
|
||||
}
|
||||
|
||||
override val gizmoduckClient: Gizmoduck.MethodPerEndpoint = {
|
||||
methodPerEndpointClient[Gizmoduck.ServicePerEndpoint, Gizmoduck.MethodPerEndpoint](
|
||||
thriftMuxClientBuilder("gizmoduck", requireMutualTls = true)
|
||||
.dest("/s/gizmoduck/gizmoduck")
|
||||
.requestTimeout(400.milliseconds)
|
||||
.timeout(900.milliseconds)
|
||||
)
|
||||
}
|
||||
|
||||
override lazy val manhattanStarbuckClient: ManhattanV1.MethodPerEndpoint = {
|
||||
methodPerEndpointClient[ManhattanV1.ServicePerEndpoint, ManhattanV1.MethodPerEndpoint](
|
||||
thriftMuxClientBuilder("manhattan.starbuck", requireMutualTls = true)
|
||||
.dest("/s/manhattan/starbuck.native-thrift")
|
||||
.requestTimeout(600.millis)
|
||||
)
|
||||
}
|
||||
|
||||
override val sgsClient: SocialGraphService.MethodPerEndpoint = {
|
||||
methodPerEndpointClient[
|
||||
SocialGraphService.ServicePerEndpoint,
|
||||
SocialGraphService.MethodPerEndpoint](
|
||||
thriftMuxClientBuilder("socialgraph", requireMutualTls = true)
|
||||
.dest("/s/socialgraph/socialgraph")
|
||||
.requestTimeout(350.milliseconds)
|
||||
.timeout(700.milliseconds)
|
||||
)
|
||||
}
|
||||
|
||||
override lazy val timelineRankerForwardingClient: TimelineRanker.FinagledClient =
|
||||
new TimelineRanker.FinagledClient(
|
||||
thriftMuxClientBuilder(
|
||||
TimelineRankerConstants.ForwardedClientName,
|
||||
ClientId(TimelineRankerConstants.ForwardedClientName),
|
||||
protocolFactoryOption = Some(new TCompactProtocol.Factory()),
|
||||
requireMutualTls = true
|
||||
).dest("/s/timelineranker/timelineranker:compactthrift").build(),
|
||||
protocolFactory = new TCompactProtocol.Factory()
|
||||
)
|
||||
|
||||
override val timelineServiceClient: TimelineService.MethodPerEndpoint = {
|
||||
methodPerEndpointClient[TimelineService.ServicePerEndpoint, TimelineService.MethodPerEndpoint](
|
||||
thriftMuxClientBuilder("timelineservice", requireMutualTls = true)
|
||||
.dest("/s/timelineservice/timelineservice")
|
||||
.requestTimeout(600.milliseconds)
|
||||
.timeout(800.milliseconds)
|
||||
)
|
||||
}
|
||||
|
||||
override val tweetyPieHighQoSClient: TweetyPie.MethodPerEndpoint = {
|
||||
methodPerEndpointClient[TweetyPie.ServicePerEndpoint, TweetyPie.MethodPerEndpoint](
|
||||
thriftMuxClientBuilder("tweetypieHighQoS", requireMutualTls = true)
|
||||
.dest("/s/tweetypie/tweetypie")
|
||||
.requestTimeout(450.milliseconds)
|
||||
.timeout(800.milliseconds),
|
||||
maxExtraLoadPercent = Some(1.percent)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Provide less costly TweetPie client with the trade-off of reduced quality. Intended for use cases
|
||||
* which are not essential for successful completion of timeline requests. Currently this client differs
|
||||
* from the highQoS endpoint by having retries count set to 1 instead of 2.
|
||||
*/
|
||||
override val tweetyPieLowQoSClient: TweetyPie.MethodPerEndpoint = {
|
||||
methodPerEndpointClient[TweetyPie.ServicePerEndpoint, TweetyPie.MethodPerEndpoint](
|
||||
thriftMuxClientBuilder("tweetypieLowQoS", requireMutualTls = true)
|
||||
.dest("/s/tweetypie/tweetypie")
|
||||
.retryPolicy(mkRetryPolicy(1)) // override default value
|
||||
.requestTimeout(450.milliseconds)
|
||||
.timeout(800.milliseconds),
|
||||
maxExtraLoadPercent = Some(1.percent)
|
||||
)
|
||||
}
|
||||
|
||||
override val userRolesServiceClient: UserRolesService.MethodPerEndpoint = {
|
||||
methodPerEndpointClient[
|
||||
UserRolesService.ServicePerEndpoint,
|
||||
UserRolesService.MethodPerEndpoint](
|
||||
thriftMuxClientBuilder("merlin", requireMutualTls = true)
|
||||
.dest("/s/merlin/merlin")
|
||||
.requestTimeout(1.second)
|
||||
)
|
||||
}
|
||||
|
||||
lazy val contentFeaturesCache: Store[TweetId, ContentFeatures] =
|
||||
new ContentFeaturesMemcacheBuilder(
|
||||
config = new StorehausMemcacheConfig(
|
||||
destName = s"$twCachePrefix/timelines_content_features:twemcaches",
|
||||
keyPrefix = "",
|
||||
requestTimeout = 50.milliseconds,
|
||||
numTries = 1,
|
||||
globalTimeout = 60.milliseconds,
|
||||
tcpConnectTimeout = 50.milliseconds,
|
||||
connectionAcquisitionTimeout = 25.milliseconds,
|
||||
numPendingRequests = 100,
|
||||
isReadOnly = false,
|
||||
serviceIdentifier = serviceIdentifier
|
||||
),
|
||||
ttl = 48.hours,
|
||||
statsReceiver
|
||||
).build
|
||||
|
||||
override val userTweetEntityGraphClient: UserTweetEntityGraph.FinagledClient =
|
||||
new UserTweetEntityGraph.FinagledClient(
|
||||
thriftMuxClientBuilder("user_tweet_entity_graph", requireMutualTls = true)
|
||||
.dest("/s/cassowary/user_tweet_entity_graph")
|
||||
.retryPolicy(mkRetryPolicy(2))
|
||||
.requestTimeout(300.milliseconds)
|
||||
.build()
|
||||
)
|
||||
|
||||
override val stratoClient: StratoClient =
|
||||
Strato.client.withMutualTls(serviceIdentifier, OpportunisticTls.Required).build()
|
||||
}
|
||||
@ -0,0 +1,13 @@
|
||||
package com.twitter.timelineranker.config
|
||||
|
||||
import com.twitter.timelines.util.stats.RequestScope
|
||||
|
||||
object RequestScopes {
|
||||
val HomeTimelineMaterialization: RequestScope = RequestScope("homeMaterialization")
|
||||
val InNetworkTweetSource: RequestScope = RequestScope("inNetworkTweet")
|
||||
val RecapHydrationSource: RequestScope = RequestScope("recapHydration")
|
||||
val RecapAuthorSource: RequestScope = RequestScope("recapAuthor")
|
||||
val ReverseChronHomeTimelineSource: RequestScope = RequestScope("reverseChronHomeTimelineSource")
|
||||
val EntityTweetsSource: RequestScope = RequestScope("entityTweets")
|
||||
val UtegLikedByTweetsSource: RequestScope = RequestScope("utegLikedByTweets")
|
||||
}
|
||||
@ -0,0 +1,133 @@
|
||||
package com.twitter.timelineranker.config
|
||||
|
||||
import com.twitter.abdecider.ABDeciderFactory
|
||||
import com.twitter.abdecider.LoggingABDecider
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.featureswitches.Value
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.decider.DeciderGateBuilder
|
||||
import com.twitter.servo.util.Effect
|
||||
import com.twitter.timelineranker.decider.DeciderKey
|
||||
import com.twitter.timelines.authorization.TimelinesClientRequestAuthorizer
|
||||
import com.twitter.timelines.config._
|
||||
import com.twitter.timelines.config.configapi._
|
||||
import com.twitter.timelines.features._
|
||||
import com.twitter.timelines.util.ImpressionCountingABDecider
|
||||
import com.twitter.timelines.util.logging.Scribe
|
||||
import com.twitter.util.Await
|
||||
import com.twitter.servo.util.Gate
|
||||
import com.twitter.timelines.model.UserId
|
||||
|
||||
trait ClientProvider {
|
||||
def clientWrappers: ClientWrappers
|
||||
def underlyingClients: UnderlyingClientConfiguration
|
||||
}
|
||||
|
||||
trait UtilityProvider {
|
||||
def abdecider: LoggingABDecider
|
||||
def clientRequestAuthorizer: TimelinesClientRequestAuthorizer
|
||||
def configStore: ConfigStore
|
||||
def decider: Decider
|
||||
def deciderGateBuilder: DeciderGateBuilder
|
||||
def employeeGate: UserRolesGate.EmployeeGate
|
||||
def configApiConfiguration: ConfigApiConfiguration
|
||||
def statsReceiver: StatsReceiver
|
||||
def whitelist: UserList
|
||||
}
|
||||
|
||||
trait RuntimeConfiguration extends ClientProvider with UtilityProvider with ConfigUtils {
|
||||
def isProd: Boolean
|
||||
def maxConcurrency: Int
|
||||
def clientEventScribe: Effect[String]
|
||||
def clientWrapperFactories: ClientWrapperFactories
|
||||
}
|
||||
|
||||
class RuntimeConfigurationImpl(
|
||||
flags: TimelineRankerFlags,
|
||||
configStoreFactory: DynamicConfigStoreFactory,
|
||||
val decider: Decider,
|
||||
val forcedFeatureValues: Map[String, Value] = Map.empty[String, Value],
|
||||
val statsReceiver: StatsReceiver)
|
||||
extends RuntimeConfiguration {
|
||||
|
||||
// Creates and initialize config store as early as possible so other parts could have a dependency on it for settings.
|
||||
override val configStore: DynamicConfigStore =
|
||||
configStoreFactory.createDcEnvAwareFileBasedConfigStore(
|
||||
relativeConfigFilePath = "timelines/timelineranker/service_settings.yml",
|
||||
dc = flags.getDatacenter,
|
||||
env = flags.getEnv,
|
||||
configBusConfig = ConfigBusProdConfig,
|
||||
onUpdate = ConfigStore.NullOnUpdateCallback,
|
||||
statsReceiver = statsReceiver
|
||||
)
|
||||
Await.result(configStore.init)
|
||||
|
||||
val environment: Env.Value = flags.getEnv
|
||||
override val isProd: Boolean = isProdEnv(environment)
|
||||
val datacenter: Datacenter.Value = flags.getDatacenter
|
||||
val abDeciderPath = "/usr/local/config/abdecider/abdecider.yml"
|
||||
override val maxConcurrency: Int = flags.maxConcurrency()
|
||||
|
||||
val deciderGateBuilder: DeciderGateBuilder = new DeciderGateBuilder(decider)
|
||||
|
||||
val clientRequestAuthorizer: TimelinesClientRequestAuthorizer =
|
||||
new TimelinesClientRequestAuthorizer(
|
||||
deciderGateBuilder = deciderGateBuilder,
|
||||
clientDetails = ClientAccessPermissions.All,
|
||||
unknownClientDetails = ClientAccessPermissions.unknown,
|
||||
clientAuthorizationGate =
|
||||
deciderGateBuilder.linearGate(DeciderKey.ClientRequestAuthorization),
|
||||
clientWriteWhitelistGate = deciderGateBuilder.linearGate(DeciderKey.ClientWriteWhitelist),
|
||||
globalCapacityQPS = flags.requestRateLimit(),
|
||||
statsReceiver = statsReceiver
|
||||
)
|
||||
override val clientEventScribe = Scribe.clientEvent(isProd, statsReceiver)
|
||||
val abdecider: LoggingABDecider = new ImpressionCountingABDecider(
|
||||
abdecider = ABDeciderFactory.withScribeEffect(
|
||||
abDeciderYmlPath = abDeciderPath,
|
||||
scribeEffect = clientEventScribe,
|
||||
decider = None,
|
||||
environment = Some("production"),
|
||||
).buildWithLogging(),
|
||||
statsReceiver = statsReceiver
|
||||
)
|
||||
|
||||
val underlyingClients: UnderlyingClientConfiguration = buildUnderlyingClientConfiguration
|
||||
|
||||
val clientWrappers: ClientWrappers = new ClientWrappers(this)
|
||||
override val clientWrapperFactories: ClientWrapperFactories = new ClientWrapperFactories(this)
|
||||
|
||||
private[this] val userRolesCacheFactory = new UserRolesCacheFactory(
|
||||
userRolesService = underlyingClients.userRolesServiceClient,
|
||||
statsReceiver = statsReceiver
|
||||
)
|
||||
override val whitelist: Whitelist = Whitelist(
|
||||
configStoreFactory = configStoreFactory,
|
||||
userRolesCacheFactory = userRolesCacheFactory,
|
||||
statsReceiver = statsReceiver
|
||||
)
|
||||
|
||||
override val employeeGate: Gate[UserId] = UserRolesGate(
|
||||
userRolesCacheFactory.create(UserRoles.EmployeesRoleName)
|
||||
)
|
||||
|
||||
private[this] val featureRecipientFactory =
|
||||
new UserRolesCachingFeatureRecipientFactory(userRolesCacheFactory, statsReceiver)
|
||||
|
||||
override val configApiConfiguration: FeatureSwitchesV2ConfigApiConfiguration =
|
||||
FeatureSwitchesV2ConfigApiConfiguration(
|
||||
datacenter = flags.getDatacenter,
|
||||
serviceName = ServiceName.TimelineRanker,
|
||||
abdecider = abdecider,
|
||||
featureRecipientFactory = featureRecipientFactory,
|
||||
forcedValues = forcedFeatureValues,
|
||||
statsReceiver = statsReceiver
|
||||
)
|
||||
|
||||
private[this] def buildUnderlyingClientConfiguration: UnderlyingClientConfiguration = {
|
||||
environment match {
|
||||
case Env.prod => new DefaultUnderlyingClientConfiguration(flags, statsReceiver)
|
||||
case _ => new StagingUnderlyingClientConfiguration(flags, statsReceiver)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,6 @@
|
||||
package com.twitter.timelineranker.config
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
|
||||
class StagingUnderlyingClientConfiguration(flags: TimelineRankerFlags, statsReceiver: StatsReceiver)
|
||||
extends DefaultUnderlyingClientConfiguration(flags, statsReceiver)
|
||||
@ -0,0 +1,8 @@
|
||||
package com.twitter.timelineranker.config
|
||||
|
||||
object TimelineRankerConstants {
|
||||
val ClientPrefix = "timelineranker."
|
||||
val ManhattanStarbuckAppId = "timelineranker"
|
||||
val WarmupClientName = "timelineranker.warmup"
|
||||
val ForwardedClientName = "timelineranker.forwarded"
|
||||
}
|
||||
@ -0,0 +1,72 @@
|
||||
package com.twitter.timelineranker.config
|
||||
|
||||
import com.twitter.app.Flags
|
||||
import com.twitter.finagle.mtls.authentication.EmptyServiceIdentifier
|
||||
import com.twitter.finagle.mtls.authentication.ServiceIdentifier
|
||||
import com.twitter.timelines.config.CommonFlags
|
||||
import com.twitter.timelines.config.ConfigUtils
|
||||
import com.twitter.timelines.config.Datacenter
|
||||
import com.twitter.timelines.config.Env
|
||||
import com.twitter.timelines.config.ProvidesServiceIdentifier
|
||||
import java.net.InetSocketAddress
|
||||
import com.twitter.app.Flag
|
||||
|
||||
class TimelineRankerFlags(flag: Flags)
|
||||
extends CommonFlags(flag)
|
||||
with ConfigUtils
|
||||
with ProvidesServiceIdentifier {
|
||||
val dc: Flag[String] = flag(
|
||||
"dc",
|
||||
"smf1",
|
||||
"Name of data center in which this instance will execute"
|
||||
)
|
||||
val environment: Flag[String] = flag(
|
||||
"environment",
|
||||
"devel",
|
||||
"The mesos environment in which this instance will be running"
|
||||
)
|
||||
val maxConcurrency: Flag[Int] = flag(
|
||||
"maxConcurrency",
|
||||
200,
|
||||
"Maximum concurrent requests"
|
||||
)
|
||||
val servicePort: Flag[InetSocketAddress] = flag(
|
||||
"service.port",
|
||||
new InetSocketAddress(8287),
|
||||
"Port number that this thrift service will listen on"
|
||||
)
|
||||
val serviceCompactPort: Flag[InetSocketAddress] = flag(
|
||||
"service.compact.port",
|
||||
new InetSocketAddress(8288),
|
||||
"Port number that the TCompactProtocol-based thrift service will listen on"
|
||||
)
|
||||
|
||||
val serviceIdentifier: Flag[ServiceIdentifier] = flag[ServiceIdentifier](
|
||||
"service.identifier",
|
||||
EmptyServiceIdentifier,
|
||||
"service identifier for this service for use with mutual TLS, " +
|
||||
"format is expected to be -service.identifier=\"role:service:environment:zone\""
|
||||
)
|
||||
|
||||
val opportunisticTlsLevel = flag[String](
|
||||
"opportunistic.tls.level",
|
||||
"desired",
|
||||
"The server's OpportunisticTls level."
|
||||
)
|
||||
|
||||
val requestRateLimit: Flag[Double] = flag[Double](
|
||||
"requestRateLimit",
|
||||
1000.0,
|
||||
"Request rate limit to be used by the client request authorizer"
|
||||
)
|
||||
|
||||
val enableThriftmuxCompression = flag(
|
||||
"enableThriftmuxServerCompression",
|
||||
true,
|
||||
"build server with thriftmux compression enabled"
|
||||
)
|
||||
|
||||
def getDatacenter: Datacenter.Value = getDC(dc())
|
||||
def getEnv: Env.Value = getEnv(environment())
|
||||
override def getServiceIdentifier: ServiceIdentifier = serviceIdentifier()
|
||||
}
|
||||
@ -0,0 +1,107 @@
|
||||
package com.twitter.timelineranker.config
|
||||
|
||||
import com.twitter.cortex_tweet_annotate.thriftscala.CortexTweetQueryService
|
||||
import com.twitter.finagle.Service
|
||||
import com.twitter.finagle.mtls.authentication.ServiceIdentifier
|
||||
import com.twitter.finagle.service.RetryPolicy
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finagle.thrift.ClientId
|
||||
import com.twitter.finagle.thrift.ThriftClientRequest
|
||||
import com.twitter.gizmoduck.thriftscala.{UserService => Gizmoduck}
|
||||
import com.twitter.manhattan.v1.thriftscala.{ManhattanCoordinator => ManhattanV1}
|
||||
import com.twitter.merlin.thriftscala.UserRolesService
|
||||
import com.twitter.recos.user_tweet_entity_graph.thriftscala.UserTweetEntityGraph
|
||||
import com.twitter.search.earlybird.thriftscala.EarlybirdService
|
||||
import com.twitter.socialgraph.thriftscala.SocialGraphService
|
||||
import com.twitter.storehaus.Store
|
||||
import com.twitter.strato.client.{Client => StratoClient}
|
||||
import com.twitter.timelineranker.recap.model.ContentFeatures
|
||||
import com.twitter.timelineranker.thriftscala.TimelineRanker
|
||||
import com.twitter.timelines.config.ConfigUtils
|
||||
import com.twitter.timelines.config.TimelinesUnderlyingClientConfiguration
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.timelineservice.thriftscala.TimelineService
|
||||
import com.twitter.tweetypie.thriftscala.{TweetService => TweetyPie}
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.Try
|
||||
import org.apache.thrift.protocol.TCompactProtocol
|
||||
|
||||
abstract class UnderlyingClientConfiguration(
|
||||
flags: TimelineRankerFlags,
|
||||
val statsReceiver: StatsReceiver)
|
||||
extends TimelinesUnderlyingClientConfiguration
|
||||
with ConfigUtils {
|
||||
|
||||
lazy val thriftClientId: ClientId = timelineRankerClientId()
|
||||
override lazy val serviceIdentifier: ServiceIdentifier = flags.getServiceIdentifier
|
||||
|
||||
def timelineRankerClientId(scope: Option[String] = None): ClientId = {
|
||||
clientIdWithScopeOpt("timelineranker", flags.getEnv, scope)
|
||||
}
|
||||
|
||||
def createEarlybirdClient(
|
||||
scope: String,
|
||||
requestTimeout: Duration,
|
||||
timeout: Duration,
|
||||
retryPolicy: RetryPolicy[Try[Nothing]]
|
||||
): EarlybirdService.MethodPerEndpoint = {
|
||||
val scopedName = s"earlybird/$scope"
|
||||
|
||||
methodPerEndpointClient[
|
||||
EarlybirdService.ServicePerEndpoint,
|
||||
EarlybirdService.MethodPerEndpoint](
|
||||
thriftMuxClientBuilder(
|
||||
scopedName,
|
||||
protocolFactoryOption = Some(new TCompactProtocol.Factory),
|
||||
requireMutualTls = true)
|
||||
.dest("/s/earlybird-root-superroot/root-superroot")
|
||||
.timeout(timeout)
|
||||
.requestTimeout(requestTimeout)
|
||||
.retryPolicy(retryPolicy)
|
||||
)
|
||||
}
|
||||
|
||||
def createEarlybirdRealtimeCgClient(
|
||||
scope: String,
|
||||
requestTimeout: Duration,
|
||||
timeout: Duration,
|
||||
retryPolicy: RetryPolicy[Try[Nothing]]
|
||||
): EarlybirdService.MethodPerEndpoint = {
|
||||
val scopedName = s"earlybird/$scope"
|
||||
|
||||
methodPerEndpointClient[
|
||||
EarlybirdService.ServicePerEndpoint,
|
||||
EarlybirdService.MethodPerEndpoint](
|
||||
thriftMuxClientBuilder(
|
||||
scopedName,
|
||||
protocolFactoryOption = Some(new TCompactProtocol.Factory),
|
||||
requireMutualTls = true)
|
||||
.dest("/s/earlybird-rootrealtimecg/root-realtime_cg")
|
||||
.timeout(timeout)
|
||||
.requestTimeout(requestTimeout)
|
||||
.retryPolicy(retryPolicy)
|
||||
)
|
||||
}
|
||||
|
||||
def cortexTweetQueryServiceClient: CortexTweetQueryService.MethodPerEndpoint
|
||||
def gizmoduckClient: Gizmoduck.MethodPerEndpoint
|
||||
def manhattanStarbuckClient: ManhattanV1.MethodPerEndpoint
|
||||
def sgsClient: SocialGraphService.MethodPerEndpoint
|
||||
def timelineRankerForwardingClient: TimelineRanker.FinagledClient
|
||||
def timelineServiceClient: TimelineService.MethodPerEndpoint
|
||||
def tweetyPieHighQoSClient: TweetyPie.MethodPerEndpoint
|
||||
def tweetyPieLowQoSClient: TweetyPie.MethodPerEndpoint
|
||||
def userRolesServiceClient: UserRolesService.MethodPerEndpoint
|
||||
def contentFeaturesCache: Store[TweetId, ContentFeatures]
|
||||
def userTweetEntityGraphClient: UserTweetEntityGraph.FinagledClient
|
||||
def stratoClient: StratoClient
|
||||
|
||||
def darkTrafficProxy: Option[Service[ThriftClientRequest, Array[Byte]]] = {
|
||||
if (flags.darkTrafficDestination.trim.nonEmpty) {
|
||||
Some(darkTrafficClient(flags.darkTrafficDestination))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,13 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"timelineranker/common/src/main/scala/com/twitter/timelineranker/model",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/core",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model",
|
||||
"timelines/src/main/scala/com/twitter/timelines/model/types",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,10 @@
|
||||
package com.twitter.timelineranker
|
||||
|
||||
import com.twitter.timelineranker.core.FutureDependencyTransformer
|
||||
import com.twitter.timelineranker.recap.model.ContentFeatures
|
||||
import com.twitter.timelines.model.TweetId
|
||||
|
||||
package object contentfeatures {
|
||||
type ContentFeaturesProvider =
|
||||
FutureDependencyTransformer[Seq[TweetId], Map[TweetId, ContentFeatures]]
|
||||
}
|
||||
@ -0,0 +1,24 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
|
||||
"src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
|
||||
"src/thrift/com/twitter/search:earlybird-scala",
|
||||
"src/thrift/com/twitter/search/common:constants-scala",
|
||||
"src/thrift/com/twitter/search/common:features-scala",
|
||||
"timelineranker/common:model",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck",
|
||||
"timelines/src/main/scala/com/twitter/timelines/model/tweet",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util",
|
||||
"timelines/src/main/scala/com/twitter/timelines/visibility/model",
|
||||
"util/util-core:util-core-util",
|
||||
],
|
||||
exports = [
|
||||
"src/thrift/com/twitter/recos/user_tweet_entity_graph:user_tweet_entity_graph-scala",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,24 @@
|
||||
package com.twitter.timelineranker.core
|
||||
|
||||
import com.twitter.recos.user_tweet_entity_graph.thriftscala.TweetRecommendation
|
||||
import com.twitter.search.earlybird.thriftscala.ThriftSearchResult
|
||||
import com.twitter.timelineranker.model.RecapQuery
|
||||
import com.twitter.timelines.model.TweetId
|
||||
|
||||
object CandidateEnvelope {
|
||||
val EmptySearchResults: Seq[ThriftSearchResult] = Seq.empty[ThriftSearchResult]
|
||||
val EmptyHydratedTweets: HydratedTweets = HydratedTweets(Seq.empty, Seq.empty)
|
||||
val EmptyUtegResults: Map[TweetId, TweetRecommendation] = Map.empty[TweetId, TweetRecommendation]
|
||||
}
|
||||
|
||||
case class CandidateEnvelope(
|
||||
query: RecapQuery,
|
||||
searchResults: Seq[ThriftSearchResult] = CandidateEnvelope.EmptySearchResults,
|
||||
utegResults: Map[TweetId, TweetRecommendation] = CandidateEnvelope.EmptyUtegResults,
|
||||
hydratedTweets: HydratedTweets = CandidateEnvelope.EmptyHydratedTweets,
|
||||
followGraphData: FollowGraphDataFuture = FollowGraphDataFuture.EmptyFollowGraphDataFuture,
|
||||
// The source tweets are
|
||||
// - the retweeted tweet, for retweets
|
||||
// - the inReplyTo tweet, for extended replies
|
||||
sourceSearchResults: Seq[ThriftSearchResult] = CandidateEnvelope.EmptySearchResults,
|
||||
sourceHydratedTweets: HydratedTweets = CandidateEnvelope.EmptyHydratedTweets)
|
||||
@ -0,0 +1,34 @@
|
||||
package com.twitter.timelineranker.core
|
||||
|
||||
import com.twitter.timelines.model.UserId
|
||||
|
||||
/**
|
||||
* Follow graph details of a given user. Includes users followed, but also followed users in various
|
||||
* states of mute.
|
||||
*
|
||||
* @param userId ID of a given user.
|
||||
* @param followedUserIds IDs of users who the given user follows.
|
||||
* @param mutuallyFollowingUserIds A subset of followedUserIds where followed users follow back the given user.
|
||||
* @param mutedUserIds A subset of followedUserIds that the given user has muted.
|
||||
* @param retweetsMutedUserIds A subset of followedUserIds whose retweets are muted by the given user.
|
||||
*/
|
||||
case class FollowGraphData(
|
||||
userId: UserId,
|
||||
followedUserIds: Seq[UserId],
|
||||
mutuallyFollowingUserIds: Set[UserId],
|
||||
mutedUserIds: Set[UserId],
|
||||
retweetsMutedUserIds: Set[UserId]) {
|
||||
val filteredFollowedUserIds: Seq[UserId] = followedUserIds.filterNot(mutedUserIds)
|
||||
val allUserIds: Seq[UserId] = filteredFollowedUserIds :+ userId
|
||||
val inNetworkUserIds: Seq[UserId] = followedUserIds :+ userId
|
||||
}
|
||||
|
||||
object FollowGraphData {
|
||||
val Empty: FollowGraphData = FollowGraphData(
|
||||
0L,
|
||||
Seq.empty[UserId],
|
||||
Set.empty[UserId],
|
||||
Set.empty[UserId],
|
||||
Set.empty[UserId]
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,53 @@
|
||||
package com.twitter.timelineranker.core
|
||||
|
||||
import com.twitter.timelines.model.UserId
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* Similar to FollowGraphData but makes available its fields as separate futures.
|
||||
*/
|
||||
case class FollowGraphDataFuture(
|
||||
userId: UserId,
|
||||
followedUserIdsFuture: Future[Seq[UserId]],
|
||||
mutuallyFollowingUserIdsFuture: Future[Set[UserId]],
|
||||
mutedUserIdsFuture: Future[Set[UserId]],
|
||||
retweetsMutedUserIdsFuture: Future[Set[UserId]]) {
|
||||
|
||||
def inNetworkUserIdsFuture: Future[Seq[UserId]] = followedUserIdsFuture.map(_ :+ userId)
|
||||
|
||||
def get(): Future[FollowGraphData] = {
|
||||
Future
|
||||
.join(
|
||||
followedUserIdsFuture,
|
||||
mutuallyFollowingUserIdsFuture,
|
||||
mutedUserIdsFuture,
|
||||
retweetsMutedUserIdsFuture
|
||||
)
|
||||
.map {
|
||||
case (followedUserIds, mutuallyFollowingUserIds, mutedUserIds, retweetsMutedUserIds) =>
|
||||
FollowGraphData(
|
||||
userId,
|
||||
followedUserIds,
|
||||
mutuallyFollowingUserIds,
|
||||
mutedUserIds,
|
||||
retweetsMutedUserIds
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object FollowGraphDataFuture {
|
||||
private def mkEmptyFuture(field: String) = {
|
||||
Future.exception(
|
||||
new IllegalStateException(s"FollowGraphDataFuture field $field accessed without being set")
|
||||
)
|
||||
}
|
||||
|
||||
val EmptyFollowGraphDataFuture: FollowGraphDataFuture = FollowGraphDataFuture(
|
||||
userId = 0L,
|
||||
followedUserIdsFuture = mkEmptyFuture("followedUserIdsFuture"),
|
||||
mutuallyFollowingUserIdsFuture = mkEmptyFuture("mutuallyFollowingUserIdsFuture"),
|
||||
mutedUserIdsFuture = mkEmptyFuture("mutedUserIdsFuture"),
|
||||
retweetsMutedUserIdsFuture = mkEmptyFuture("retweetsMutedUserIdsFuture")
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,18 @@
|
||||
package com.twitter.timelineranker.core
|
||||
|
||||
import com.twitter.search.common.constants.thriftscala.ThriftLanguage
|
||||
import com.twitter.search.common.features.thriftscala.ThriftTweetFeatures
|
||||
import com.twitter.timelineranker.recap.model.ContentFeatures
|
||||
import com.twitter.timelines.clients.gizmoduck.UserProfileInfo
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.timelines.util.FutureUtils
|
||||
import com.twitter.util.Future
|
||||
|
||||
case class HydratedCandidatesAndFeaturesEnvelope(
|
||||
candidateEnvelope: CandidateEnvelope,
|
||||
userLanguages: Seq[ThriftLanguage],
|
||||
userProfileInfo: UserProfileInfo,
|
||||
features: Map[TweetId, ThriftTweetFeatures] = Map.empty,
|
||||
contentFeaturesFuture: Future[Map[TweetId, ContentFeatures]] = FutureUtils.EmptyMap,
|
||||
tweetSourceTweetMap: Map[TweetId, TweetId] = Map.empty,
|
||||
inReplyToTweetIds: Set[TweetId] = Set.empty)
|
||||
@ -0,0 +1,7 @@
|
||||
package com.twitter.timelineranker.core
|
||||
|
||||
import com.twitter.timelines.model.tweet.HydratedTweet
|
||||
|
||||
case class HydratedTweets(
|
||||
outerTweets: Seq[HydratedTweet],
|
||||
innerTweets: Seq[HydratedTweet] = Seq.empty)
|
||||
@ -0,0 +1,13 @@
|
||||
package com.twitter.timelineranker
|
||||
|
||||
import com.twitter.timelineranker.model.RecapQuery
|
||||
import com.twitter.timelines.configapi
|
||||
|
||||
package object core {
|
||||
type FutureDependencyTransformer[-U, +V] = configapi.FutureDependencyTransformer[RecapQuery, U, V]
|
||||
object FutureDependencyTransformer
|
||||
extends configapi.FutureDependencyTransformerFunctions[RecapQuery]
|
||||
|
||||
type DependencyTransformer[-U, +V] = configapi.DependencyTransformer[RecapQuery, U, V]
|
||||
object DependencyTransformer extends configapi.DependencyTransformerFunctions[RecapQuery]
|
||||
}
|
||||
@ -0,0 +1,9 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"servo/decider",
|
||||
"timelineranker/server/config",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,83 @@
|
||||
package com.twitter.timelineranker.decider
|
||||
|
||||
import com.twitter.servo.decider.DeciderKeyEnum
|
||||
|
||||
object DeciderKey extends DeciderKeyEnum {
|
||||
// Deciders that can be used to control load on TLR or its backends.
|
||||
val EnableMaxConcurrencyLimiting: Value = Value("enable_max_concurrency_limiting")
|
||||
|
||||
// Deciders related to testing / debugging.
|
||||
val EnableRoutingToRankerDevProxy: Value = Value("enable_routing_to_ranker_dev_proxy")
|
||||
|
||||
// Deciders related to authorization.
|
||||
val ClientRequestAuthorization: Value = Value("client_request_authorization")
|
||||
val ClientWriteWhitelist: Value = Value("client_write_whitelist")
|
||||
val AllowTimelineMixerRecapProd: Value = Value("allow_timeline_mixer_recap_prod")
|
||||
val AllowTimelineMixerRecycledProd: Value = Value("allow_timeline_mixer_recycled_prod")
|
||||
val AllowTimelineMixerHydrateProd: Value = Value("allow_timeline_mixer_hydrate_prod")
|
||||
val AllowTimelineMixerHydrateRecosProd: Value = Value("allow_timeline_mixer_hydrate_recos_prod")
|
||||
val AllowTimelineMixerSeedAuthorsProd: Value = Value("allow_timeline_mixer_seed_authors_prod")
|
||||
val AllowTimelineMixerSimclusterProd: Value = Value("allow_timeline_mixer_simcluster_prod")
|
||||
val AllowTimelineMixerEntityTweetsProd: Value = Value("allow_timeline_mixer_entity_tweets_prod")
|
||||
val AllowTimelineMixerListProd: Value = Value("allow_timeline_mixer_list_prod")
|
||||
val AllowTimelineMixerListTweetProd: Value = Value("allow_timeline_mixer_list_tweet_prod")
|
||||
val AllowTimelineMixerCommunityProd: Value = Value("allow_timeline_mixer_community_prod")
|
||||
val AllowTimelineMixerCommunityTweetProd: Value = Value(
|
||||
"allow_timeline_mixer_community_tweet_prod")
|
||||
val AllowTimelineScorerRecommendedTrendTweetProd: Value = Value(
|
||||
"allow_timeline_scorer_recommended_trend_tweet_prod")
|
||||
val AllowTimelineMixerUtegLikedByTweetsProd: Value = Value(
|
||||
"allow_timeline_mixer_uteg_liked_by_tweets_prod")
|
||||
val AllowTimelineMixerStaging: Value = Value("allow_timeline_mixer_staging")
|
||||
val AllowTimelineRankerProxy: Value = Value("allow_timeline_ranker_proxy")
|
||||
val AllowTimelineRankerWarmup: Value = Value("allow_timeline_ranker_warmup")
|
||||
val AllowTimelineScorerRecTopicTweetsProd: Value =
|
||||
Value("allow_timeline_scorer_rec_topic_tweets_prod")
|
||||
val AllowTimelineScorerPopularTopicTweetsProd: Value =
|
||||
Value("allow_timeline_scorer_popular_topic_tweets_prod")
|
||||
val AllowTimelineScorerHydrateTweetScoringProd: Value = Value(
|
||||
"allow_timelinescorer_hydrate_tweet_scoring_prod")
|
||||
val AllowTimelineServiceProd: Value = Value("allow_timeline_service_prod")
|
||||
val AllowTimelineServiceStaging: Value = Value("allow_timeline_service_staging")
|
||||
val RateLimitOverrideUnknown: Value = Value("rate_limit_override_unknown")
|
||||
|
||||
// Deciders related to reverse-chron home timeline materialization.
|
||||
val MultiplierOfMaterializationTweetsFetched: Value = Value(
|
||||
"multiplier_of_materialization_tweets_fetched"
|
||||
)
|
||||
val BackfillFilteredEntries: Value = Value("enable_backfill_filtered_entries")
|
||||
val TweetsFilteringLossageThreshold: Value = Value("tweets_filtering_lossage_threshold")
|
||||
val TweetsFilteringLossageLimit: Value = Value("tweets_filtering_lossage_limit")
|
||||
val SupplementFollowsWithRealGraph: Value = Value("supplement_follows_with_real_graph")
|
||||
|
||||
// Deciders related to recap.
|
||||
val RecapEnableContentFeaturesHydration: Value = Value("recap_enable_content_features_hydration")
|
||||
val RecapMaxCountMultiplier: Value = Value("recap_max_count_multiplier")
|
||||
val RecapEnableExtraSortingInResults: Value = Value("recap_enable_extra_sorting_in_results")
|
||||
|
||||
// Deciders related to recycled tweets.
|
||||
val RecycledMaxCountMultiplier: Value = Value("recycled_max_count_multiplier")
|
||||
val RecycledEnableContentFeaturesHydration: Value = Value(
|
||||
"recycled_enable_content_features_hydration")
|
||||
|
||||
// Deciders related to entity tweets.
|
||||
val EntityTweetsEnableContentFeaturesHydration: Value = Value(
|
||||
"entity_tweets_enable_content_features_hydration")
|
||||
|
||||
// Deciders related to both recap and recycled tweets
|
||||
val EnableRealGraphUsers: Value = Value("enable_real_graph_users")
|
||||
val MaxRealGraphAndFollowedUsers: Value = Value("max_real_graph_and_followed_users")
|
||||
|
||||
// Deciders related to recap author
|
||||
val RecapAuthorEnableNewPipeline: Value = Value("recap_author_enable_new_pipeline")
|
||||
val RecapAuthorEnableContentFeaturesHydration: Value = Value(
|
||||
"recap_author_enable_content_features_hydration")
|
||||
|
||||
// Deciders related to recap hydration (rectweet and ranked organic).
|
||||
val RecapHydrationEnableContentFeaturesHydration: Value = Value(
|
||||
"recap_hydration_enable_content_features_hydration")
|
||||
|
||||
// Deciders related to uteg liked by tweets
|
||||
val UtegLikedByTweetsEnableContentFeaturesHydration: Value = Value(
|
||||
"uteg_liked_by_tweets_enable_content_features_hydration")
|
||||
}
|
||||
@ -0,0 +1,39 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/twitter/storehaus:core",
|
||||
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
|
||||
"configapi/configapi-decider/src/main/scala/com/twitter/timelines/configapi/decider",
|
||||
"finagle/finagle-core/src/main",
|
||||
"servo/util/src/main/scala",
|
||||
"src/thrift/com/twitter/search:earlybird-scala",
|
||||
"timelineranker/common/src/main/scala/com/twitter/timelineranker/model",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/common",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/config",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/core",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/repository",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/util",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/visibility",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/manhattan",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/relevance_search",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/tweetypie",
|
||||
"timelines/src/main/scala/com/twitter/timelines/common/model",
|
||||
"timelines/src/main/scala/com/twitter/timelines/config",
|
||||
"timelines/src/main/scala/com/twitter/timelines/earlybird/common/options",
|
||||
"timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util/stats",
|
||||
"timelines/src/main/scala/com/twitter/timelines/visibility",
|
||||
"timelines/src/main/scala/com/twitter/timelines/visibility/model",
|
||||
"util/util-core:util-core-util",
|
||||
"util/util-core/src/main/scala/com/twitter/conversions",
|
||||
"util/util-stats/src/main/scala",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,20 @@
|
||||
package com.twitter.timelineranker.entity_tweets
|
||||
|
||||
import com.twitter.timelineranker.model.CandidateTweetsResult
|
||||
import com.twitter.timelineranker.model.RecapQuery
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* A repository of entity tweets candidates.
|
||||
*
|
||||
* For now, it does not cache any results therefore forwards all calls to the underlying source.
|
||||
*/
|
||||
class EntityTweetsRepository(source: EntityTweetsSource) {
|
||||
def get(query: RecapQuery): Future[CandidateTweetsResult] = {
|
||||
source.get(query)
|
||||
}
|
||||
|
||||
def get(queries: Seq[RecapQuery]): Future[Seq[CandidateTweetsResult]] = {
|
||||
source.get(queries)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,60 @@
|
||||
package com.twitter.timelineranker.entity_tweets
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.timelineranker.config.RequestScopes
|
||||
import com.twitter.timelineranker.config.RuntimeConfiguration
|
||||
import com.twitter.timelineranker.parameters.ConfigBuilder
|
||||
import com.twitter.timelineranker.repository.CandidatesRepositoryBuilder
|
||||
import com.twitter.timelineranker.visibility.SgsFollowGraphDataFields
|
||||
import com.twitter.search.earlybird.thriftscala.EarlybirdService
|
||||
import com.twitter.timelines.util.stats.RequestScope
|
||||
import com.twitter.util.Duration
|
||||
|
||||
class EntityTweetsRepositoryBuilder(config: RuntimeConfiguration, configBuilder: ConfigBuilder)
|
||||
extends CandidatesRepositoryBuilder(config) {
|
||||
|
||||
// Default client id for this repository if the upstream requests doesn't indicate one.
|
||||
override val clientSubId = "community"
|
||||
override val requestScope: RequestScope = RequestScopes.EntityTweetsSource
|
||||
override val followGraphDataFieldsToFetch: SgsFollowGraphDataFields.ValueSet =
|
||||
SgsFollowGraphDataFields.ValueSet(
|
||||
SgsFollowGraphDataFields.FollowedUserIds,
|
||||
SgsFollowGraphDataFields.MutuallyFollowingUserIds,
|
||||
SgsFollowGraphDataFields.MutedUserIds
|
||||
)
|
||||
|
||||
/**
|
||||
* [1] timeout is derived from p9999 TLR <-> Earlybird latency and shall be less than
|
||||
* request timeout of timelineranker client within downstream timelinemixer, which is
|
||||
* 1s now
|
||||
*
|
||||
* [2] processing timeout is less than request timeout [1] with 100ms space for networking and
|
||||
* other times such as gc
|
||||
*/
|
||||
override val searchProcessingTimeout: Duration = 550.milliseconds // [2]
|
||||
override def earlybirdClient(scope: String): EarlybirdService.MethodPerEndpoint =
|
||||
config.underlyingClients.createEarlybirdClient(
|
||||
scope = scope,
|
||||
requestTimeout = 650.milliseconds, // [1]
|
||||
timeout = 900.milliseconds, // [1]
|
||||
retryPolicy = config.underlyingClients.DefaultRetryPolicy
|
||||
)
|
||||
|
||||
def apply(): EntityTweetsRepository = {
|
||||
val entityTweetsSource = new EntityTweetsSource(
|
||||
gizmoduckClient,
|
||||
searchClient,
|
||||
tweetyPieHighQoSClient,
|
||||
userMetadataClient,
|
||||
followGraphDataProvider,
|
||||
clientFactories.visibilityEnforcerFactory.apply(
|
||||
VisibilityRules,
|
||||
RequestScopes.EntityTweetsSource
|
||||
),
|
||||
config.underlyingClients.contentFeaturesCache,
|
||||
config.statsReceiver
|
||||
)
|
||||
|
||||
new EntityTweetsRepository(entityTweetsSource)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,71 @@
|
||||
package com.twitter.timelineranker.entity_tweets
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.model.TweetIdRange
|
||||
import com.twitter.timelines.clients.relevance_search.SearchClient
|
||||
import com.twitter.timelines.clients.relevance_search.SearchClient.TweetTypes
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.util.Future
|
||||
|
||||
object EntityTweetsSearchResultsTransform {
|
||||
// If EntityTweetsQuery.maxCount is not specified, the following count is used.
|
||||
val DefaultEntityTweetsMaxTweetCount = 200
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch entity tweets search results using the search client
|
||||
* and populate them into the CandidateEnvelope
|
||||
*/
|
||||
class EntityTweetsSearchResultsTransform(
|
||||
searchClient: SearchClient,
|
||||
statsReceiver: StatsReceiver,
|
||||
logSearchDebugInfo: Boolean = false)
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
import EntityTweetsSearchResultsTransform._
|
||||
|
||||
private[this] val maxCountStat = statsReceiver.stat("maxCount")
|
||||
private[this] val numResultsFromSearchStat = statsReceiver.stat("numResultsFromSearch")
|
||||
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
val maxCount = envelope.query.maxCount.getOrElse(DefaultEntityTweetsMaxTweetCount)
|
||||
maxCountStat.add(maxCount)
|
||||
|
||||
val tweetIdRange = envelope.query.range
|
||||
.map(TweetIdRange.fromTimelineRange)
|
||||
.getOrElse(TweetIdRange.default)
|
||||
|
||||
val beforeTweetIdExclusive = tweetIdRange.toId
|
||||
val afterTweetIdExclusive = tweetIdRange.fromId
|
||||
|
||||
val excludedTweetIds = envelope.query.excludedTweetIds.getOrElse(Seq.empty[TweetId]).toSet
|
||||
val languages = envelope.query.languages.map(_.map(_.language))
|
||||
|
||||
envelope.followGraphData.inNetworkUserIdsFuture.flatMap { inNetworkUserIds =>
|
||||
searchClient
|
||||
.getEntityTweets(
|
||||
userId = Some(envelope.query.userId),
|
||||
followedUserIds = inNetworkUserIds.toSet,
|
||||
maxCount = maxCount,
|
||||
beforeTweetIdExclusive = beforeTweetIdExclusive,
|
||||
afterTweetIdExclusive = afterTweetIdExclusive,
|
||||
earlybirdOptions = envelope.query.earlybirdOptions,
|
||||
semanticCoreIds = envelope.query.semanticCoreIds,
|
||||
hashtags = envelope.query.hashtags,
|
||||
languages = languages,
|
||||
tweetTypes = TweetTypes.fromTweetKindOption(envelope.query.options),
|
||||
searchOperator = envelope.query.searchOperator,
|
||||
excludedTweetIds = excludedTweetIds,
|
||||
logSearchDebugInfo = logSearchDebugInfo,
|
||||
includeNullcastTweets = envelope.query.includeNullcastTweets.getOrElse(false),
|
||||
includeTweetsFromArchiveIndex =
|
||||
envelope.query.includeTweetsFromArchiveIndex.getOrElse(false),
|
||||
authorIds = envelope.query.authorIds.map(_.toSet)
|
||||
).map { results =>
|
||||
numResultsFromSearchStat.add(results.size)
|
||||
envelope.copy(searchResults = results)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,146 @@
|
||||
package com.twitter.timelineranker.entity_tweets
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.storehaus.Store
|
||||
import com.twitter.timelineranker.common._
|
||||
import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
|
||||
import com.twitter.timelineranker.model._
|
||||
import com.twitter.timelineranker.parameters.entity_tweets.EntityTweetsParams._
|
||||
import com.twitter.timelineranker.recap.model.ContentFeatures
|
||||
import com.twitter.timelineranker.util.CopyContentFeaturesIntoHydratedTweetsTransform
|
||||
import com.twitter.timelineranker.util.CopyContentFeaturesIntoThriftTweetFeaturesTransform
|
||||
import com.twitter.timelineranker.util.TweetFilters
|
||||
import com.twitter.timelineranker.visibility.FollowGraphDataProvider
|
||||
import com.twitter.timelines.clients.gizmoduck.GizmoduckClient
|
||||
import com.twitter.timelines.clients.manhattan.UserMetadataClient
|
||||
import com.twitter.timelines.clients.relevance_search.SearchClient
|
||||
import com.twitter.timelines.clients.tweetypie.TweetyPieClient
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.timelines.util.FailOpenHandler
|
||||
import com.twitter.timelines.util.stats.RequestStatsReceiver
|
||||
import com.twitter.timelines.visibility.VisibilityEnforcer
|
||||
import com.twitter.util.Future
|
||||
|
||||
class EntityTweetsSource(
|
||||
gizmoduckClient: GizmoduckClient,
|
||||
searchClient: SearchClient,
|
||||
tweetyPieClient: TweetyPieClient,
|
||||
userMetadataClient: UserMetadataClient,
|
||||
followGraphDataProvider: FollowGraphDataProvider,
|
||||
visibilityEnforcer: VisibilityEnforcer,
|
||||
contentFeaturesCache: Store[TweetId, ContentFeatures],
|
||||
statsReceiver: StatsReceiver) {
|
||||
|
||||
private[this] val baseScope = statsReceiver.scope("entityTweetsSource")
|
||||
private[this] val requestStats = RequestStatsReceiver(baseScope)
|
||||
|
||||
private[this] val failOpenScope = baseScope.scope("failOpen")
|
||||
private[this] val userProfileHandler = new FailOpenHandler(failOpenScope, "userProfileInfo")
|
||||
private[this] val userLanguagesHandler = new FailOpenHandler(failOpenScope, "userLanguages")
|
||||
|
||||
private[this] val followGraphDataTransform = new FollowGraphDataTransform(
|
||||
followGraphDataProvider = followGraphDataProvider,
|
||||
maxFollowedUsersProvider = DependencyProvider.from(MaxFollowedUsersParam)
|
||||
)
|
||||
private[this] val fetchSearchResultsTransform = new EntityTweetsSearchResultsTransform(
|
||||
searchClient = searchClient,
|
||||
statsReceiver = baseScope
|
||||
)
|
||||
private[this] val userProfileInfoTransform =
|
||||
new UserProfileInfoTransform(userProfileHandler, gizmoduckClient)
|
||||
private[this] val languagesTransform =
|
||||
new UserLanguagesTransform(userLanguagesHandler, userMetadataClient)
|
||||
|
||||
private[this] val visibilityEnforcingTransform = new VisibilityEnforcingTransform(
|
||||
visibilityEnforcer
|
||||
)
|
||||
|
||||
private[this] val filters = TweetFilters.ValueSet(
|
||||
TweetFilters.DuplicateTweets,
|
||||
TweetFilters.DuplicateRetweets
|
||||
)
|
||||
|
||||
private[this] val hydratedTweetsFilter = new HydratedTweetsFilterTransform(
|
||||
outerFilters = filters,
|
||||
innerFilters = TweetFilters.None,
|
||||
useFollowGraphData = false,
|
||||
useSourceTweets = false,
|
||||
statsReceiver = baseScope,
|
||||
numRetweetsAllowed = HydratedTweetsFilterTransform.NumDuplicateRetweetsAllowed
|
||||
)
|
||||
|
||||
private[this] val contentFeaturesHydrationTransform =
|
||||
new ContentFeaturesHydrationTransformBuilder(
|
||||
tweetyPieClient = tweetyPieClient,
|
||||
contentFeaturesCache = contentFeaturesCache,
|
||||
enableContentFeaturesGate = RecapQuery.paramGate(EnableContentFeaturesHydrationParam),
|
||||
enableTokensInContentFeaturesGate =
|
||||
RecapQuery.paramGate(EnableTokensInContentFeaturesHydrationParam),
|
||||
enableTweetTextInContentFeaturesGate =
|
||||
RecapQuery.paramGate(EnableTweetTextInContentFeaturesHydrationParam),
|
||||
enableConversationControlContentFeaturesGate =
|
||||
RecapQuery.paramGate(EnableConversationControlInContentFeaturesHydrationParam),
|
||||
enableTweetMediaHydrationGate = RecapQuery.paramGate(EnableTweetMediaHydrationParam),
|
||||
hydrateInReplyToTweets = false,
|
||||
statsReceiver = baseScope
|
||||
).build()
|
||||
|
||||
private[this] def hydratesContentFeatures(
|
||||
hydratedEnvelope: HydratedCandidatesAndFeaturesEnvelope
|
||||
): Boolean =
|
||||
hydratedEnvelope.candidateEnvelope.query.hydratesContentFeatures.getOrElse(true)
|
||||
|
||||
private[this] val contentFeaturesTransformer = FutureArrow.choose(
|
||||
predicate = hydratesContentFeatures,
|
||||
ifTrue = contentFeaturesHydrationTransform
|
||||
.andThen(CopyContentFeaturesIntoThriftTweetFeaturesTransform)
|
||||
.andThen(CopyContentFeaturesIntoHydratedTweetsTransform),
|
||||
ifFalse = FutureArrow[
|
||||
HydratedCandidatesAndFeaturesEnvelope,
|
||||
HydratedCandidatesAndFeaturesEnvelope
|
||||
](Future.value) // empty transformer
|
||||
)
|
||||
|
||||
private[this] val candidateGenerationTransform = new CandidateGenerationTransform(baseScope)
|
||||
|
||||
private[this] val hydrationAndFilteringPipeline =
|
||||
CreateCandidateEnvelopeTransform
|
||||
.andThen(followGraphDataTransform) // Fetch follow graph data
|
||||
.andThen(fetchSearchResultsTransform) // fetch search results
|
||||
.andThen(SearchResultDedupAndSortingTransform) // dedup and order search results
|
||||
.andThen(CandidateTweetHydrationTransform) // hydrate search results
|
||||
.andThen(visibilityEnforcingTransform) // filter hydrated tweets to visible ones
|
||||
.andThen(hydratedTweetsFilter) // filter hydrated tweets based on predefined filter
|
||||
.andThen(
|
||||
TrimToMatchHydratedTweetsTransform
|
||||
) // trim search result set to match filtered hydrated tweets (this needs to be accurate for feature hydration)
|
||||
|
||||
// runs the main pipeline in parallel with fetching user profile info and user languages
|
||||
private[this] val featureHydrationDataTransform =
|
||||
new FeatureHydrationDataTransform(
|
||||
hydrationAndFilteringPipeline,
|
||||
languagesTransform,
|
||||
userProfileInfoTransform
|
||||
)
|
||||
|
||||
private[this] val tweetFeaturesHydrationTransform =
|
||||
OutOfNetworkTweetsSearchFeaturesHydrationTransform
|
||||
.andThen(contentFeaturesTransformer)
|
||||
|
||||
private[this] val featureHydrationPipeline =
|
||||
featureHydrationDataTransform
|
||||
.andThen(tweetFeaturesHydrationTransform)
|
||||
.andThen(candidateGenerationTransform)
|
||||
|
||||
def get(query: RecapQuery): Future[CandidateTweetsResult] = {
|
||||
requestStats.addEventStats {
|
||||
featureHydrationPipeline(query)
|
||||
}
|
||||
}
|
||||
|
||||
def get(queries: Seq[RecapQuery]): Future[Seq[CandidateTweetsResult]] = {
|
||||
Future.collect(queries.map(get))
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/twitter/storehaus:core",
|
||||
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
|
||||
"configapi/configapi-decider/src/main/scala/com/twitter/timelines/configapi/decider",
|
||||
"finagle/finagle-core/src/main",
|
||||
"servo/util/src/main/scala",
|
||||
"src/thrift/com/twitter/search:earlybird-scala",
|
||||
"timelineranker/common/src/main/scala/com/twitter/timelineranker/model",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/common",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/config",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/core",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/monitoring",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/recap/model",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/repository",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/util",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/visibility",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/gizmoduck",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/manhattan",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/relevance_search",
|
||||
"timelines/src/main/scala/com/twitter/timelines/clients/tweetypie",
|
||||
"timelines/src/main/scala/com/twitter/timelines/config",
|
||||
"timelines/src/main/scala/com/twitter/timelines/earlybird/common/utils",
|
||||
"timelines/src/main/scala/com/twitter/timelines/model/tweet",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util/stats",
|
||||
"timelines/src/main/scala/com/twitter/timelines/visibility",
|
||||
"timelines/src/main/scala/com/twitter/timelines/visibility/model",
|
||||
"util/util-core:util-core-util",
|
||||
"util/util-core/src/main/scala/com/twitter/conversions",
|
||||
"util/util-stats/src/main/scala",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,31 @@
|
||||
package com.twitter.timelineranker.in_network_tweets
|
||||
|
||||
import com.twitter.timelineranker.model.CandidateTweetsResult
|
||||
import com.twitter.timelineranker.model.RecapQuery
|
||||
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
|
||||
import com.twitter.timelineranker.parameters.in_network_tweets.InNetworkTweetParams
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* A repository of in-network tweet candidates.
|
||||
* For now, it does not cache any results therefore forwards all calls to the underlying source.
|
||||
*/
|
||||
class InNetworkTweetRepository(
|
||||
source: InNetworkTweetSource,
|
||||
realtimeCGSource: InNetworkTweetSource) {
|
||||
|
||||
private[this] val enableRealtimeCGProvider =
|
||||
DependencyProvider.from(InNetworkTweetParams.EnableEarlybirdRealtimeCgMigrationParam)
|
||||
|
||||
def get(query: RecapQuery): Future[CandidateTweetsResult] = {
|
||||
if (enableRealtimeCGProvider(query)) {
|
||||
realtimeCGSource.get(query)
|
||||
} else {
|
||||
source.get(query)
|
||||
}
|
||||
}
|
||||
|
||||
def get(queries: Seq[RecapQuery]): Future[Seq[CandidateTweetsResult]] = {
|
||||
Future.collect(queries.map(query => get(query)))
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,109 @@
|
||||
package com.twitter.timelineranker.in_network_tweets
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.finagle.service.RetryPolicy
|
||||
import com.twitter.search.earlybird.thriftscala.EarlybirdService
|
||||
import com.twitter.timelineranker.config.RequestScopes
|
||||
import com.twitter.timelineranker.config.RuntimeConfiguration
|
||||
import com.twitter.timelineranker.parameters.ConfigBuilder
|
||||
import com.twitter.timelineranker.repository.CandidatesRepositoryBuilder
|
||||
import com.twitter.timelineranker.visibility.SgsFollowGraphDataFields
|
||||
import com.twitter.timelines.util.stats.RequestScope
|
||||
import com.twitter.timelines.visibility.model.CheckedUserActorType
|
||||
import com.twitter.timelines.visibility.model.ExclusionReason
|
||||
import com.twitter.timelines.visibility.model.VisibilityCheckStatus
|
||||
import com.twitter.timelines.visibility.model.VisibilityCheckUser
|
||||
import com.twitter.util.Duration
|
||||
|
||||
object InNetworkTweetRepositoryBuilder {
|
||||
val VisibilityRuleExclusions: Set[ExclusionReason] = Set[ExclusionReason](
|
||||
ExclusionReason(
|
||||
CheckedUserActorType(Some(false), VisibilityCheckUser.SourceUser),
|
||||
Set(VisibilityCheckStatus.Blocked)
|
||||
)
|
||||
)
|
||||
|
||||
private val EarlybirdTimeout = 600.milliseconds
|
||||
private val EarlybirdRequestTimeout = 600.milliseconds
|
||||
|
||||
/**
|
||||
* The timeouts below are only used for the Earlybird Cluster Migration
|
||||
*/
|
||||
private val EarlybirdRealtimeCGTimeout = 600.milliseconds
|
||||
private val EarlybirdRealtimeCGRequestTimeout = 600.milliseconds
|
||||
}
|
||||
|
||||
class InNetworkTweetRepositoryBuilder(config: RuntimeConfiguration, configBuilder: ConfigBuilder)
|
||||
extends CandidatesRepositoryBuilder(config) {
|
||||
import InNetworkTweetRepositoryBuilder._
|
||||
|
||||
override val clientSubId = "recycled_tweets"
|
||||
override val requestScope: RequestScope = RequestScopes.InNetworkTweetSource
|
||||
override val followGraphDataFieldsToFetch: SgsFollowGraphDataFields.ValueSet =
|
||||
SgsFollowGraphDataFields.ValueSet(
|
||||
SgsFollowGraphDataFields.FollowedUserIds,
|
||||
SgsFollowGraphDataFields.MutuallyFollowingUserIds,
|
||||
SgsFollowGraphDataFields.MutedUserIds,
|
||||
SgsFollowGraphDataFields.RetweetsMutedUserIds
|
||||
)
|
||||
override val searchProcessingTimeout: Duration = 200.milliseconds
|
||||
|
||||
override def earlybirdClient(scope: String): EarlybirdService.MethodPerEndpoint =
|
||||
config.underlyingClients.createEarlybirdClient(
|
||||
scope = scope,
|
||||
requestTimeout = EarlybirdRequestTimeout,
|
||||
timeout = EarlybirdTimeout,
|
||||
retryPolicy = RetryPolicy.Never
|
||||
)
|
||||
|
||||
private lazy val searchClientForSourceTweets =
|
||||
newSearchClient(clientId = clientSubId + "_source_tweets")
|
||||
|
||||
/** The RealtimeCG clients below are only used for the Earlybird Cluster Migration */
|
||||
private def earlybirdRealtimeCGClient(scope: String): EarlybirdService.MethodPerEndpoint =
|
||||
config.underlyingClients.createEarlybirdRealtimeCgClient(
|
||||
scope = scope,
|
||||
requestTimeout = EarlybirdRealtimeCGRequestTimeout,
|
||||
timeout = EarlybirdRealtimeCGTimeout,
|
||||
retryPolicy = RetryPolicy.Never
|
||||
)
|
||||
private val realtimeCGClientSubId = "realtime_cg_recycled_tweets"
|
||||
private lazy val searchRealtimeCGClient =
|
||||
newSearchClient(earlybirdRealtimeCGClient, clientId = realtimeCGClientSubId)
|
||||
|
||||
def apply(): InNetworkTweetRepository = {
|
||||
val inNetworkTweetSource = new InNetworkTweetSource(
|
||||
gizmoduckClient,
|
||||
searchClient,
|
||||
searchClientForSourceTweets,
|
||||
tweetyPieHighQoSClient,
|
||||
userMetadataClient,
|
||||
followGraphDataProvider,
|
||||
config.underlyingClients.contentFeaturesCache,
|
||||
clientFactories.visibilityEnforcerFactory.apply(
|
||||
VisibilityRules,
|
||||
RequestScopes.InNetworkTweetSource,
|
||||
reasonsToExclude = InNetworkTweetRepositoryBuilder.VisibilityRuleExclusions
|
||||
),
|
||||
config.statsReceiver
|
||||
)
|
||||
|
||||
val inNetworkTweetRealtimeCGSource = new InNetworkTweetSource(
|
||||
gizmoduckClient,
|
||||
searchRealtimeCGClient,
|
||||
searchClientForSourceTweets, // do not migrate source_tweets as they are sharded by TweetID
|
||||
tweetyPieHighQoSClient,
|
||||
userMetadataClient,
|
||||
followGraphDataProvider,
|
||||
config.underlyingClients.contentFeaturesCache,
|
||||
clientFactories.visibilityEnforcerFactory.apply(
|
||||
VisibilityRules,
|
||||
RequestScopes.InNetworkTweetSource,
|
||||
reasonsToExclude = InNetworkTweetRepositoryBuilder.VisibilityRuleExclusions
|
||||
),
|
||||
config.statsReceiver.scope("replacementRealtimeCG")
|
||||
)
|
||||
|
||||
new InNetworkTweetRepository(inNetworkTweetSource, inNetworkTweetRealtimeCGSource)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,271 @@
|
||||
package com.twitter.timelineranker.in_network_tweets
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.storehaus.Store
|
||||
import com.twitter.timelineranker.common._
|
||||
import com.twitter.timelineranker.core.HydratedCandidatesAndFeaturesEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
|
||||
import com.twitter.timelineranker.model._
|
||||
import com.twitter.timelineranker.monitoring.UsersSearchResultMonitoringTransform
|
||||
import com.twitter.timelineranker.parameters.in_network_tweets.InNetworkTweetParams
|
||||
import com.twitter.timelineranker.parameters.monitoring.MonitoringParams
|
||||
import com.twitter.timelineranker.parameters.recap.RecapParams
|
||||
import com.twitter.timelineranker.recap.model.ContentFeatures
|
||||
import com.twitter.timelineranker.util.CopyContentFeaturesIntoHydratedTweetsTransform
|
||||
import com.twitter.timelineranker.util.CopyContentFeaturesIntoThriftTweetFeaturesTransform
|
||||
import com.twitter.timelineranker.util.TweetFilters
|
||||
import com.twitter.timelineranker.visibility.FollowGraphDataProvider
|
||||
import com.twitter.timelines.clients.gizmoduck.GizmoduckClient
|
||||
import com.twitter.timelines.clients.manhattan.UserMetadataClient
|
||||
import com.twitter.timelines.clients.relevance_search.SearchClient
|
||||
import com.twitter.timelines.clients.tweetypie.TweetyPieClient
|
||||
import com.twitter.timelines.model.TweetId
|
||||
import com.twitter.timelines.util.FailOpenHandler
|
||||
import com.twitter.timelines.util.stats.RequestStatsReceiver
|
||||
import com.twitter.timelines.visibility.VisibilityEnforcer
|
||||
import com.twitter.util.Future
|
||||
|
||||
class InNetworkTweetSource(
|
||||
gizmoduckClient: GizmoduckClient,
|
||||
searchClient: SearchClient,
|
||||
searchClientForSourceTweets: SearchClient,
|
||||
tweetyPieClient: TweetyPieClient,
|
||||
userMetadataClient: UserMetadataClient,
|
||||
followGraphDataProvider: FollowGraphDataProvider,
|
||||
contentFeaturesCache: Store[TweetId, ContentFeatures],
|
||||
visibilityEnforcer: VisibilityEnforcer,
|
||||
statsReceiver: StatsReceiver) {
|
||||
private[this] val baseScope = statsReceiver.scope("recycledTweetSource")
|
||||
private[this] val requestStats = RequestStatsReceiver(baseScope)
|
||||
|
||||
private[this] val failOpenScope = baseScope.scope("failOpen")
|
||||
private[this] val userProfileHandler = new FailOpenHandler(failOpenScope, "userProfileInfo")
|
||||
private[this] val userLanguagesHandler = new FailOpenHandler(failOpenScope, "userLanguages")
|
||||
private[this] val sourceTweetSearchHandler =
|
||||
new FailOpenHandler(failOpenScope, "sourceTweetSearch")
|
||||
|
||||
private[this] val filters = TweetFilters.ValueSet(
|
||||
TweetFilters.DuplicateTweets,
|
||||
TweetFilters.DuplicateRetweets,
|
||||
TweetFilters.TweetsFromNotFollowedUsers,
|
||||
TweetFilters.NonReplyDirectedAtNotFollowedUsers
|
||||
)
|
||||
|
||||
private[this] val hydrateReplyRootTweetProvider =
|
||||
DependencyProvider.from(InNetworkTweetParams.EnableReplyRootTweetHydrationParam)
|
||||
|
||||
private[this] val sourceTweetsSearchResultsTransform = new SourceTweetsSearchResultsTransform(
|
||||
searchClientForSourceTweets,
|
||||
sourceTweetSearchHandler,
|
||||
hydrateReplyRootTweetProvider = hydrateReplyRootTweetProvider,
|
||||
perRequestSourceSearchClientIdProvider = DependencyProvider.None,
|
||||
baseScope
|
||||
)
|
||||
|
||||
private[this] val visibilityEnforcingTransform = new VisibilityEnforcingTransform(
|
||||
visibilityEnforcer
|
||||
)
|
||||
|
||||
private[this] val hydratedTweetsFilter = new HydratedTweetsFilterTransform(
|
||||
outerFilters = filters,
|
||||
innerFilters = TweetFilters.None,
|
||||
useFollowGraphData = true,
|
||||
useSourceTweets = true,
|
||||
statsReceiver = baseScope,
|
||||
numRetweetsAllowed = HydratedTweetsFilterTransform.NumDuplicateRetweetsAllowed
|
||||
)
|
||||
|
||||
private[this] val dynamicHydratedTweetsFilter = new TweetKindOptionHydratedTweetsFilterTransform(
|
||||
useFollowGraphData = true,
|
||||
useSourceTweets = true,
|
||||
statsReceiver = baseScope
|
||||
)
|
||||
|
||||
private[this] val userProfileInfoTransform =
|
||||
new UserProfileInfoTransform(userProfileHandler, gizmoduckClient)
|
||||
private[this] val languagesTransform =
|
||||
new UserLanguagesTransform(userLanguagesHandler, userMetadataClient)
|
||||
|
||||
private[this] def hydratesContentFeatures(
|
||||
hydratedEnvelope: HydratedCandidatesAndFeaturesEnvelope
|
||||
): Boolean =
|
||||
hydratedEnvelope.candidateEnvelope.query.hydratesContentFeatures.getOrElse(true)
|
||||
|
||||
private[this] val contentFeaturesTransformer = FutureArrow.choose(
|
||||
predicate = hydratesContentFeatures,
|
||||
ifTrue = contentFeaturesHydrationTransform
|
||||
.andThen(CopyContentFeaturesIntoThriftTweetFeaturesTransform)
|
||||
.andThen(CopyContentFeaturesIntoHydratedTweetsTransform),
|
||||
ifFalse = FutureArrow[
|
||||
HydratedCandidatesAndFeaturesEnvelope,
|
||||
HydratedCandidatesAndFeaturesEnvelope
|
||||
](Future.value) // empty transformer
|
||||
)
|
||||
|
||||
private[this] val contentFeaturesHydrationTransform =
|
||||
new ContentFeaturesHydrationTransformBuilder(
|
||||
tweetyPieClient = tweetyPieClient,
|
||||
contentFeaturesCache = contentFeaturesCache,
|
||||
enableContentFeaturesGate =
|
||||
RecapQuery.paramGate(InNetworkTweetParams.EnableContentFeaturesHydrationParam),
|
||||
enableTokensInContentFeaturesGate =
|
||||
RecapQuery.paramGate(InNetworkTweetParams.EnableTokensInContentFeaturesHydrationParam),
|
||||
enableTweetTextInContentFeaturesGate =
|
||||
RecapQuery.paramGate(InNetworkTweetParams.EnableTweetTextInContentFeaturesHydrationParam),
|
||||
enableConversationControlContentFeaturesGate = RecapQuery.paramGate(
|
||||
InNetworkTweetParams.EnableConversationControlInContentFeaturesHydrationParam),
|
||||
enableTweetMediaHydrationGate = RecapQuery.paramGate(
|
||||
InNetworkTweetParams.EnableTweetMediaHydrationParam
|
||||
),
|
||||
hydrateInReplyToTweets = true,
|
||||
statsReceiver = baseScope
|
||||
).build()
|
||||
|
||||
private[this] val candidateGenerationTransform = new CandidateGenerationTransform(baseScope)
|
||||
|
||||
private[this] val maxFollowedUsersProvider =
|
||||
DependencyProvider.from(InNetworkTweetParams.MaxFollowedUsersParam)
|
||||
private[this] val earlybirdReturnAllResultsProvider =
|
||||
DependencyProvider.from(InNetworkTweetParams.EnableEarlybirdReturnAllResultsParam)
|
||||
private[this] val relevanceOptionsMaxHitsToProcessProvider =
|
||||
DependencyProvider.from(InNetworkTweetParams.RelevanceOptionsMaxHitsToProcessParam)
|
||||
|
||||
private[this] val followGraphDataTransform =
|
||||
new FollowGraphDataTransform(followGraphDataProvider, maxFollowedUsersProvider)
|
||||
|
||||
private[this] val enableRealGraphUsersProvider =
|
||||
DependencyProvider.from(RecapParams.EnableRealGraphUsersParam)
|
||||
private[this] val maxRealGraphAndFollowedUsersProvider =
|
||||
DependencyProvider.from(RecapParams.MaxRealGraphAndFollowedUsersParam)
|
||||
private[this] val maxRealGraphAndFollowedUsersFSOverrideProvider =
|
||||
DependencyProvider.from(RecapParams.MaxRealGraphAndFollowedUsersFSOverrideParam)
|
||||
private[this] val imputeRealGraphAuthorWeightsProvider =
|
||||
DependencyProvider.from(RecapParams.ImputeRealGraphAuthorWeightsParam)
|
||||
private[this] val imputeRealGraphAuthorWeightsPercentileProvider =
|
||||
DependencyProvider.from(RecapParams.ImputeRealGraphAuthorWeightsPercentileParam)
|
||||
private[this] val maxRealGraphAndFollowedUsersFromDeciderAndFS = DependencyProvider { envelope =>
|
||||
maxRealGraphAndFollowedUsersFSOverrideProvider(envelope).getOrElse(
|
||||
maxRealGraphAndFollowedUsersProvider(envelope))
|
||||
}
|
||||
private[this] val followAndRealGraphCombiningTransform = new FollowAndRealGraphCombiningTransform(
|
||||
followGraphDataProvider = followGraphDataProvider,
|
||||
maxFollowedUsersProvider = maxFollowedUsersProvider,
|
||||
enableRealGraphUsersProvider = enableRealGraphUsersProvider,
|
||||
maxRealGraphAndFollowedUsersProvider = maxRealGraphAndFollowedUsersFromDeciderAndFS,
|
||||
imputeRealGraphAuthorWeightsProvider = imputeRealGraphAuthorWeightsProvider,
|
||||
imputeRealGraphAuthorWeightsPercentileProvider = imputeRealGraphAuthorWeightsPercentileProvider,
|
||||
statsReceiver = baseScope
|
||||
)
|
||||
|
||||
private[this] val maxCountProvider = DependencyProvider { query =>
|
||||
query.maxCount.getOrElse(query.params(InNetworkTweetParams.DefaultMaxTweetCount))
|
||||
}
|
||||
|
||||
private[this] val maxCountWithMarginProvider = DependencyProvider { query =>
|
||||
val maxCount = query.maxCount.getOrElse(query.params(InNetworkTweetParams.DefaultMaxTweetCount))
|
||||
val multiplier = query.params(InNetworkTweetParams.MaxCountMultiplierParam)
|
||||
(maxCount * multiplier).toInt
|
||||
}
|
||||
|
||||
private[this] val debugAuthorsMonitoringProvider =
|
||||
DependencyProvider.from(MonitoringParams.DebugAuthorsAllowListParam)
|
||||
|
||||
private[this] val retrieveSearchResultsTransform = new RecapSearchResultsTransform(
|
||||
searchClient = searchClient,
|
||||
maxCountProvider = maxCountWithMarginProvider,
|
||||
returnAllResultsProvider = earlybirdReturnAllResultsProvider,
|
||||
relevanceOptionsMaxHitsToProcessProvider = relevanceOptionsMaxHitsToProcessProvider,
|
||||
enableExcludeSourceTweetIdsProvider = DependencyProvider.True,
|
||||
enableSettingTweetTypesWithTweetKindOptionProvider =
|
||||
DependencyProvider.from(RecapParams.EnableSettingTweetTypesWithTweetKindOption),
|
||||
perRequestSearchClientIdProvider = DependencyProvider.None,
|
||||
statsReceiver = baseScope,
|
||||
logSearchDebugInfo = false
|
||||
)
|
||||
|
||||
private[this] val preTruncateSearchResultsTransform =
|
||||
new UsersSearchResultMonitoringTransform(
|
||||
name = "RecapSearchResultsTruncationTransform",
|
||||
new RecapSearchResultsTruncationTransform(
|
||||
extraSortBeforeTruncationGate = DependencyProvider.True,
|
||||
maxCountProvider = maxCountWithMarginProvider,
|
||||
statsReceiver = baseScope.scope("afterSearchResultsTransform")
|
||||
),
|
||||
baseScope.scope("afterSearchResultsTransform"),
|
||||
debugAuthorsMonitoringProvider
|
||||
)
|
||||
|
||||
private[this] val finalTruncationTransform = new UsersSearchResultMonitoringTransform(
|
||||
name = "RecapSearchResultsTruncationTransform",
|
||||
new RecapSearchResultsTruncationTransform(
|
||||
extraSortBeforeTruncationGate = DependencyProvider.True,
|
||||
maxCountProvider = maxCountProvider,
|
||||
statsReceiver = baseScope.scope("finalTruncation")
|
||||
),
|
||||
baseScope.scope("finalTruncation"),
|
||||
debugAuthorsMonitoringProvider
|
||||
)
|
||||
|
||||
// Fetch source tweets based on search results present in the envelope
|
||||
// and hydrate them.
|
||||
private[this] val fetchAndHydrateSourceTweets =
|
||||
sourceTweetsSearchResultsTransform
|
||||
.andThen(SourceTweetHydrationTransform)
|
||||
|
||||
// Hydrate candidate tweets and fetch source tweets in parallel
|
||||
private[this] val hydrateTweetsAndSourceTweetsInParallel =
|
||||
new HydrateTweetsAndSourceTweetsInParallelTransform(
|
||||
candidateTweetHydration = CandidateTweetHydrationTransform,
|
||||
sourceTweetHydration = fetchAndHydrateSourceTweets
|
||||
)
|
||||
|
||||
private[this] val trimToMatchSearchResultsTransform = new TrimToMatchSearchResultsTransform(
|
||||
hydrateReplyRootTweetProvider = hydrateReplyRootTweetProvider,
|
||||
statsReceiver = baseScope
|
||||
)
|
||||
|
||||
private[this] val hydrationAndFilteringPipeline =
|
||||
CreateCandidateEnvelopeTransform // Create empty CandidateEnvelope
|
||||
.andThen(followGraphDataTransform) // Fetch follow graph data
|
||||
.andThen(followAndRealGraphCombiningTransform) // Experiment: expand seed author set
|
||||
.andThen(retrieveSearchResultsTransform) // Fetch search results
|
||||
.andThen(
|
||||
preTruncateSearchResultsTransform
|
||||
) // truncate the search result up to maxCount + some margin, preserving the random tweet
|
||||
.andThen(SearchResultDedupAndSortingTransform) // dedups, and sorts reverse-chron
|
||||
.andThen(hydrateTweetsAndSourceTweetsInParallel) // candidates + source tweets in parallel
|
||||
.andThen(visibilityEnforcingTransform) // filter hydrated tweets to visible ones
|
||||
.andThen(hydratedTweetsFilter) // filter hydrated tweets based on predefined filter
|
||||
.andThen(dynamicHydratedTweetsFilter) // filter hydrated tweets based on query TweetKindOption
|
||||
.andThen(TrimToMatchHydratedTweetsTransform) // trim searchResult to match with hydratedTweets
|
||||
.andThen(
|
||||
finalTruncationTransform
|
||||
) // truncate the searchResult to exactly up to maxCount, preserving the random tweet
|
||||
.andThen(
|
||||
trimToMatchSearchResultsTransform
|
||||
) // trim other fields to match with the final searchResult
|
||||
|
||||
// runs the main pipeline in parallel with fetching user profile info and user languages
|
||||
private[this] val featureHydrationDataTransform = new FeatureHydrationDataTransform(
|
||||
hydrationAndFilteringPipeline,
|
||||
languagesTransform,
|
||||
userProfileInfoTransform
|
||||
)
|
||||
|
||||
private[this] val featureHydrationPipeline =
|
||||
featureHydrationDataTransform
|
||||
.andThen(InNetworkTweetsSearchFeaturesHydrationTransform)
|
||||
.andThen(contentFeaturesTransformer)
|
||||
.andThen(candidateGenerationTransform)
|
||||
|
||||
def get(query: RecapQuery): Future[CandidateTweetsResult] = {
|
||||
requestStats.addEventStats {
|
||||
featureHydrationPipeline(query)
|
||||
}
|
||||
}
|
||||
|
||||
def get(queries: Seq[RecapQuery]): Future[Seq[CandidateTweetsResult]] = {
|
||||
Future.collect(queries.map(get))
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,15 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
platform = "java8",
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/org/apache/thrift:libthrift",
|
||||
"scrooge/scrooge-core/src/main/scala",
|
||||
"servo/util",
|
||||
"src/thrift/com/twitter/search:earlybird-scala",
|
||||
"timelineranker/common/src/main/scala/com/twitter/timelineranker/model",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/core",
|
||||
"util/util-stats/src/main/scala/com/twitter/finagle/stats",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,52 @@
|
||||
package com.twitter.timelineranker.monitoring
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.search.earlybird.thriftscala.ThriftSearchResult
|
||||
import com.twitter.servo.util.FutureArrow
|
||||
import com.twitter.timelineranker.core.CandidateEnvelope
|
||||
import com.twitter.timelineranker.model.RecapQuery.DependencyProvider
|
||||
import com.twitter.util.Future
|
||||
|
||||
/**
|
||||
* Captures tweet counts pre and post transformation for a list of users
|
||||
*/
|
||||
class UsersSearchResultMonitoringTransform(
|
||||
name: String,
|
||||
underlyingTransformer: FutureArrow[CandidateEnvelope, CandidateEnvelope],
|
||||
statsReceiver: StatsReceiver,
|
||||
debugAuthorListDependencyProvider: DependencyProvider[Seq[Long]])
|
||||
extends FutureArrow[CandidateEnvelope, CandidateEnvelope] {
|
||||
|
||||
private val scopedStatsReceiver = statsReceiver.scope(name)
|
||||
private val preTransformCounter = (userId: Long) =>
|
||||
scopedStatsReceiver
|
||||
.scope("pre_transform").scope(userId.toString).counter("debug_author_list")
|
||||
private val postTransformCounter = (userId: Long) =>
|
||||
scopedStatsReceiver
|
||||
.scope("post_transform").scope(userId.toString).counter("debug_author_list")
|
||||
|
||||
override def apply(envelope: CandidateEnvelope): Future[CandidateEnvelope] = {
|
||||
val debugAuthorList = debugAuthorListDependencyProvider.apply(envelope.query)
|
||||
envelope.searchResults
|
||||
.filter(isTweetFromDebugAuthorList(_, debugAuthorList))
|
||||
.flatMap(_.metadata)
|
||||
.foreach(metadata => preTransformCounter(metadata.fromUserId).incr())
|
||||
|
||||
underlyingTransformer
|
||||
.apply(envelope)
|
||||
.map { result =>
|
||||
envelope.searchResults
|
||||
.filter(isTweetFromDebugAuthorList(_, debugAuthorList))
|
||||
.flatMap(_.metadata)
|
||||
.foreach(metadata => postTransformCounter(metadata.fromUserId).incr())
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
private def isTweetFromDebugAuthorList(
|
||||
searchResult: ThriftSearchResult,
|
||||
debugAuthorList: Seq[Long]
|
||||
): Boolean =
|
||||
searchResult.metadata.exists(metadata => debugAuthorList.contains(metadata.fromUserId))
|
||||
|
||||
}
|
||||
@ -0,0 +1,15 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"src/thrift/com/twitter/timelineranker/server/model:thrift-scala",
|
||||
"timelineranker/common:model",
|
||||
"timelines:observe",
|
||||
"timelines/src/main/scala/com/twitter/timelines/authorization",
|
||||
"timelines/src/main/scala/com/twitter/timelines/features",
|
||||
"timelines/src/main/scala/com/twitter/timelines/model/types",
|
||||
"util/util-core:util-core-util",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,36 @@
|
||||
package com.twitter.timelineranker.observe
|
||||
|
||||
import com.twitter.servo.util.Gate
|
||||
import com.twitter.timelineranker.model.TimelineQuery
|
||||
import com.twitter.timelines.features.Features
|
||||
import com.twitter.timelines.features.UserList
|
||||
import com.twitter.timelines.observe.DebugObserver
|
||||
import com.twitter.timelineranker.{thriftscala => thrift}
|
||||
|
||||
/**
|
||||
* Builds the DebugObserver that is attached to thrift requests.
|
||||
* This class exists to centralize the gates that determine whether or not
|
||||
* to enable debug transcripts for a particular request.
|
||||
*/
|
||||
class DebugObserverBuilder(whitelist: UserList) {
|
||||
|
||||
lazy val observer: DebugObserver = build()
|
||||
|
||||
private[this] def build(): DebugObserver = {
|
||||
new DebugObserver(queryGate)
|
||||
}
|
||||
|
||||
private[observe] def queryGate: Gate[Any] = {
|
||||
val shouldEnableDebug = whitelist.userIdGate(Features.DebugTranscript)
|
||||
|
||||
Gate { a: Any =>
|
||||
a match {
|
||||
case q: thrift.EngagedTweetsQuery => shouldEnableDebug(q.userId)
|
||||
case q: thrift.RecapHydrationQuery => shouldEnableDebug(q.userId)
|
||||
case q: thrift.RecapQuery => shouldEnableDebug(q.userId)
|
||||
case q: TimelineQuery => shouldEnableDebug(q.userId)
|
||||
case _ => false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,35 @@
|
||||
package com.twitter.timelineranker.observe
|
||||
|
||||
import com.twitter.timelines.authorization.ReadRequest
|
||||
import com.twitter.timelines.model.UserId
|
||||
import com.twitter.timelines.observe.ObservedAndValidatedRequests
|
||||
import com.twitter.timelines.observe.ServiceObserver
|
||||
import com.twitter.timelines.observe.ServiceTracer
|
||||
import com.twitter.util.Future
|
||||
|
||||
trait ObservedRequests extends ObservedAndValidatedRequests {
|
||||
|
||||
def observeAndValidate[R, Q](
|
||||
request: Q,
|
||||
viewerIds: Seq[UserId],
|
||||
stats: ServiceObserver.Stats[Q],
|
||||
exceptionHandler: PartialFunction[Throwable, Future[R]]
|
||||
)(
|
||||
f: Q => Future[R]
|
||||
): Future[R] = {
|
||||
super.observeAndValidate[Q, R](
|
||||
request,
|
||||
viewerIds,
|
||||
ReadRequest,
|
||||
validateRequest,
|
||||
exceptionHandler,
|
||||
stats,
|
||||
ServiceTracer.identity[Q]
|
||||
)(f)
|
||||
}
|
||||
|
||||
def validateRequest[Q](request: Q): Unit = {
|
||||
// TimelineQuery and its derived classes do not permit invalid instances to be constructed.
|
||||
// Therefore no additional validation is required.
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,18 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"configapi/configapi-core",
|
||||
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
|
||||
"servo/decider",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/entity_tweets",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/in_network_tweets",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/monitoring",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_author",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap_hydration",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/revchron",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/uteg_liked_by_tweets",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,60 @@
|
||||
package com.twitter.timelineranker.parameters
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.decider.DeciderGateBuilder
|
||||
import com.twitter.timelineranker.parameters.entity_tweets.EntityTweetsProduction
|
||||
import com.twitter.timelineranker.parameters.recap.RecapProduction
|
||||
import com.twitter.timelineranker.parameters.recap_author.RecapAuthorProduction
|
||||
import com.twitter.timelineranker.parameters.recap_hydration.RecapHydrationProduction
|
||||
import com.twitter.timelineranker.parameters.in_network_tweets.InNetworkTweetProduction
|
||||
import com.twitter.timelineranker.parameters.revchron.ReverseChronProduction
|
||||
import com.twitter.timelineranker.parameters.uteg_liked_by_tweets.UtegLikedByTweetsProduction
|
||||
import com.twitter.timelineranker.parameters.monitoring.MonitoringProduction
|
||||
import com.twitter.timelines.configapi.CompositeConfig
|
||||
import com.twitter.timelines.configapi.Config
|
||||
|
||||
/**
|
||||
* Builds global composite config containing prioritized "layers" of parameter overrides
|
||||
* based on whitelists, experiments, and deciders. Generated config can be used in tests with
|
||||
* mocked decider and whitelist.
|
||||
*/
|
||||
class ConfigBuilder(deciderGateBuilder: DeciderGateBuilder, statsReceiver: StatsReceiver) {
|
||||
|
||||
/**
|
||||
* Production config which includes all configs which contribute to production behavior. At
|
||||
* minimum, it should include all configs containing decider-based param overrides.
|
||||
*
|
||||
* It is important that the production config include all production param overrides as it is
|
||||
* used to build holdback experiment configs; If the production config doesn't include all param
|
||||
* overrides supporting production behavior then holdback experiment "production" buckets will
|
||||
* not reflect production behavior.
|
||||
*/
|
||||
val prodConfig: Config = new CompositeConfig(
|
||||
Seq(
|
||||
new RecapProduction(deciderGateBuilder, statsReceiver).config,
|
||||
new InNetworkTweetProduction(deciderGateBuilder).config,
|
||||
new ReverseChronProduction(deciderGateBuilder).config,
|
||||
new EntityTweetsProduction(deciderGateBuilder).config,
|
||||
new RecapAuthorProduction(deciderGateBuilder).config,
|
||||
new RecapHydrationProduction(deciderGateBuilder).config,
|
||||
new UtegLikedByTweetsProduction(deciderGateBuilder).config,
|
||||
MonitoringProduction.config
|
||||
),
|
||||
"prodConfig"
|
||||
)
|
||||
|
||||
val whitelistConfig: Config = new CompositeConfig(
|
||||
Seq(
|
||||
// No whitelists configured at present.
|
||||
),
|
||||
"whitelistConfig"
|
||||
)
|
||||
|
||||
val rootConfig: Config = new CompositeConfig(
|
||||
Seq(
|
||||
whitelistConfig,
|
||||
prodConfig
|
||||
),
|
||||
"rootConfig"
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,12 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
|
||||
"configapi/configapi-decider/src/main/scala/com/twitter/timelines/configapi/decider",
|
||||
"servo/decider",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/decider",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,65 @@
|
||||
package com.twitter.timelineranker.parameters.entity_tweets
|
||||
|
||||
import com.twitter.timelineranker.decider.DeciderKey
|
||||
import com.twitter.timelines.configapi.decider.DeciderParam
|
||||
import com.twitter.timelines.configapi.FSBoundedParam
|
||||
import com.twitter.timelines.configapi.FSParam
|
||||
|
||||
object EntityTweetsParams {
|
||||
|
||||
/**
|
||||
* Controls limit on the number of followed users fetched from SGS.
|
||||
*/
|
||||
object MaxFollowedUsersParam
|
||||
extends FSBoundedParam[Int](
|
||||
name = "entity_tweets_max_followed_users",
|
||||
default = 1000,
|
||||
min = 0,
|
||||
max = 5000
|
||||
)
|
||||
|
||||
/**
|
||||
* Enables semantic core, penguin, and tweetypie content features in entity tweets source.
|
||||
*/
|
||||
object EnableContentFeaturesHydrationParam
|
||||
extends DeciderParam[Boolean](
|
||||
decider = DeciderKey.EntityTweetsEnableContentFeaturesHydration,
|
||||
default = false
|
||||
)
|
||||
|
||||
/**
|
||||
* additionally enables tokens when hydrating content features.
|
||||
*/
|
||||
object EnableTokensInContentFeaturesHydrationParam
|
||||
extends FSParam(
|
||||
name = "entity_tweets_enable_tokens_in_content_features_hydration",
|
||||
default = false
|
||||
)
|
||||
|
||||
/**
|
||||
* additionally enables tweet text when hydrating content features.
|
||||
* This only works if EnableContentFeaturesHydrationParam is set to true
|
||||
*/
|
||||
object EnableTweetTextInContentFeaturesHydrationParam
|
||||
extends FSParam(
|
||||
name = "entity_tweets_enable_tweet_text_in_content_features_hydration",
|
||||
default = false
|
||||
)
|
||||
|
||||
/**
|
||||
* additionally enables conversationControl when hydrating content features.
|
||||
* This only works if EnableContentFeaturesHydrationParam is set to true
|
||||
*/
|
||||
object EnableConversationControlInContentFeaturesHydrationParam
|
||||
extends FSParam(
|
||||
name = "conversation_control_in_content_features_hydration_entity_enable",
|
||||
default = false
|
||||
)
|
||||
|
||||
object EnableTweetMediaHydrationParam
|
||||
extends FSParam(
|
||||
name = "tweet_media_hydration_entity_tweets_enable",
|
||||
default = false
|
||||
)
|
||||
|
||||
}
|
||||
@ -0,0 +1,42 @@
|
||||
package com.twitter.timelineranker.parameters.entity_tweets
|
||||
|
||||
import com.twitter.servo.decider.DeciderGateBuilder
|
||||
import com.twitter.servo.decider.DeciderKeyName
|
||||
import com.twitter.timelineranker.decider.DeciderKey
|
||||
import com.twitter.timelineranker.parameters.entity_tweets.EntityTweetsParams._
|
||||
import com.twitter.timelines.configapi.decider.DeciderUtils
|
||||
import com.twitter.timelines.configapi.BaseConfig
|
||||
import com.twitter.timelines.configapi.BaseConfigBuilder
|
||||
import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil
|
||||
import com.twitter.timelines.configapi.Param
|
||||
|
||||
object EntityTweetsProduction {
|
||||
val deciderByParam: Map[Param[_], DeciderKeyName] = Map[Param[_], DeciderKeyName](
|
||||
EnableContentFeaturesHydrationParam -> DeciderKey.EntityTweetsEnableContentFeaturesHydration
|
||||
)
|
||||
}
|
||||
|
||||
case class EntityTweetsProduction(deciderGateBuilder: DeciderGateBuilder) {
|
||||
|
||||
val booleanDeciderOverrides = DeciderUtils.getBooleanDeciderOverrides(
|
||||
deciderGateBuilder,
|
||||
EnableContentFeaturesHydrationParam
|
||||
)
|
||||
|
||||
val booleanFeatureSwitchOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides(
|
||||
EnableTokensInContentFeaturesHydrationParam,
|
||||
EnableTweetTextInContentFeaturesHydrationParam,
|
||||
EnableConversationControlInContentFeaturesHydrationParam,
|
||||
EnableTweetMediaHydrationParam
|
||||
)
|
||||
|
||||
val intFeatureSwitchOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides(
|
||||
MaxFollowedUsersParam
|
||||
)
|
||||
|
||||
val config: BaseConfig = new BaseConfigBuilder()
|
||||
.set(booleanDeciderOverrides: _*)
|
||||
.set(booleanFeatureSwitchOverrides: _*)
|
||||
.set(intFeatureSwitchOverrides: _*)
|
||||
.build()
|
||||
}
|
||||
@ -0,0 +1,15 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
|
||||
"configapi/configapi-decider/src/main/scala/com/twitter/timelines/configapi/decider",
|
||||
"servo/decider",
|
||||
"servo/util/src/main/scala",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/decider",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/recap",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,133 @@
|
||||
package com.twitter.timelineranker.parameters.in_network_tweets
|
||||
|
||||
import com.twitter.timelineranker.parameters.recap.RecapQueryContext
|
||||
import com.twitter.timelines.configapi.decider._
|
||||
import com.twitter.timelines.configapi.FSBoundedParam
|
||||
import com.twitter.timelines.configapi.FSParam
|
||||
import com.twitter.timelines.configapi.Param
|
||||
|
||||
object InNetworkTweetParams {
|
||||
import RecapQueryContext._
|
||||
|
||||
/**
|
||||
* Controls limit on the number of followed users fetched from SGS.
|
||||
*
|
||||
* The specific default value below is for blender-timelines parity.
|
||||
*/
|
||||
object MaxFollowedUsersParam
|
||||
extends FSBoundedParam[Int](
|
||||
name = "recycled_max_followed_users",
|
||||
default = MaxFollowedUsers.default,
|
||||
min = MaxFollowedUsers.bounds.minInclusive,
|
||||
max = MaxFollowedUsers.bounds.maxInclusive
|
||||
)
|
||||
|
||||
/**
|
||||
* Controls limit on the number of hits for Earlybird.
|
||||
*
|
||||
*/
|
||||
object RelevanceOptionsMaxHitsToProcessParam
|
||||
extends FSBoundedParam[Int](
|
||||
name = "recycled_relevance_options_max_hits_to_process",
|
||||
default = 500,
|
||||
min = 100,
|
||||
max = 20000
|
||||
)
|
||||
|
||||
/**
|
||||
* Fallback value for maximum number of search results, if not specified by query.maxCount
|
||||
*/
|
||||
object DefaultMaxTweetCount extends Param(200)
|
||||
|
||||
/**
|
||||
* We multiply maxCount (caller supplied value) by this multiplier and fetch those many
|
||||
* candidates from search so that we are left with sufficient number of candidates after
|
||||
* hydration and filtering.
|
||||
*/
|
||||
object MaxCountMultiplierParam
|
||||
extends Param(MaxCountMultiplier.default)
|
||||
with DeciderValueConverter[Double] {
|
||||
override def convert: IntConverter[Double] =
|
||||
OutputBoundIntConverter[Double](divideDeciderBy100 _, MaxCountMultiplier.bounds)
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable [[SearchQueryBuilder.createExcludedSourceTweetIdsQuery]]
|
||||
*/
|
||||
object EnableExcludeSourceTweetIdsQueryParam
|
||||
extends FSParam[Boolean](
|
||||
name = "recycled_exclude_source_tweet_ids_query_enable",
|
||||
default = false
|
||||
)
|
||||
|
||||
object EnableEarlybirdReturnAllResultsParam
|
||||
extends FSParam[Boolean](
|
||||
name = "recycled_enable_earlybird_return_all_results",
|
||||
default = true
|
||||
)
|
||||
|
||||
/**
|
||||
* FS-controlled param to enable anti-dilution transform for DDG-16198
|
||||
*/
|
||||
object RecycledMaxFollowedUsersEnableAntiDilutionParam
|
||||
extends FSParam[Boolean](
|
||||
name = "recycled_max_followed_users_enable_anti_dilution",
|
||||
default = false
|
||||
)
|
||||
|
||||
/**
|
||||
* Enables semantic core, penguin, and tweetypie content features in recycled source.
|
||||
*/
|
||||
object EnableContentFeaturesHydrationParam extends Param(default = true)
|
||||
|
||||
/**
|
||||
* additionally enables tokens when hydrating content features.
|
||||
*/
|
||||
object EnableTokensInContentFeaturesHydrationParam
|
||||
extends FSParam(
|
||||
name = "recycled_enable_tokens_in_content_features_hydration",
|
||||
default = false
|
||||
)
|
||||
|
||||
/**
|
||||
* additionally enables tweet text when hydrating content features.
|
||||
* This only works if EnableContentFeaturesHydrationParam is set to true
|
||||
*/
|
||||
object EnableTweetTextInContentFeaturesHydrationParam
|
||||
extends FSParam(
|
||||
name = "recycled_enable_tweet_text_in_content_features_hydration",
|
||||
default = false
|
||||
)
|
||||
|
||||
/**
|
||||
* Enables hydrating root tweet of in-network replies and extended replies
|
||||
*/
|
||||
object EnableReplyRootTweetHydrationParam
|
||||
extends FSParam(
|
||||
name = "recycled_enable_reply_root_tweet_hydration",
|
||||
default = true
|
||||
)
|
||||
|
||||
/**
|
||||
* additionally enables conversationControl when hydrating content features.
|
||||
* This only works if EnableContentFeaturesHydrationParam is set to true
|
||||
*/
|
||||
object EnableConversationControlInContentFeaturesHydrationParam
|
||||
extends FSParam(
|
||||
name = "conversation_control_in_content_features_hydration_recycled_enable",
|
||||
default = false
|
||||
)
|
||||
|
||||
object EnableTweetMediaHydrationParam
|
||||
extends FSParam(
|
||||
name = "tweet_media_hydration_recycled_enable",
|
||||
default = false
|
||||
)
|
||||
|
||||
object EnableEarlybirdRealtimeCgMigrationParam
|
||||
extends FSParam(
|
||||
name = "recycled_enable_earlybird_realtime_cg_migration",
|
||||
default = false
|
||||
)
|
||||
|
||||
}
|
||||
@ -0,0 +1,71 @@
|
||||
package com.twitter.timelineranker.parameters.in_network_tweets
|
||||
|
||||
import com.twitter.servo.decider.DeciderGateBuilder
|
||||
import com.twitter.servo.decider.DeciderKeyName
|
||||
import com.twitter.timelineranker.decider.DeciderKey
|
||||
import com.twitter.timelineranker.parameters.in_network_tweets.InNetworkTweetParams._
|
||||
import com.twitter.timelineranker.parameters.util.ConfigHelper
|
||||
import com.twitter.timelines.configapi._
|
||||
import com.twitter.servo.decider.DeciderKeyEnum
|
||||
|
||||
object InNetworkTweetProduction {
|
||||
val deciderByParam: Map[Param[_], DeciderKeyEnum#Value] = Map[Param[_], DeciderKeyName](
|
||||
EnableContentFeaturesHydrationParam -> DeciderKey.RecycledEnableContentFeaturesHydration,
|
||||
MaxCountMultiplierParam -> DeciderKey.RecycledMaxCountMultiplier
|
||||
)
|
||||
|
||||
val doubleParams: Seq[MaxCountMultiplierParam.type] = Seq(
|
||||
MaxCountMultiplierParam
|
||||
)
|
||||
|
||||
val booleanDeciderParams: Seq[EnableContentFeaturesHydrationParam.type] = Seq(
|
||||
EnableContentFeaturesHydrationParam
|
||||
)
|
||||
|
||||
val booleanFeatureSwitchParams: Seq[FSParam[Boolean]] = Seq(
|
||||
EnableExcludeSourceTweetIdsQueryParam,
|
||||
EnableTokensInContentFeaturesHydrationParam,
|
||||
EnableReplyRootTweetHydrationParam,
|
||||
EnableTweetTextInContentFeaturesHydrationParam,
|
||||
EnableConversationControlInContentFeaturesHydrationParam,
|
||||
EnableTweetMediaHydrationParam,
|
||||
EnableEarlybirdReturnAllResultsParam,
|
||||
EnableEarlybirdRealtimeCgMigrationParam,
|
||||
RecycledMaxFollowedUsersEnableAntiDilutionParam
|
||||
)
|
||||
|
||||
val boundedIntFeatureSwitchParams: Seq[FSBoundedParam[Int]] = Seq(
|
||||
MaxFollowedUsersParam,
|
||||
RelevanceOptionsMaxHitsToProcessParam
|
||||
)
|
||||
}
|
||||
|
||||
class InNetworkTweetProduction(deciderGateBuilder: DeciderGateBuilder) {
|
||||
val configHelper: ConfigHelper =
|
||||
new ConfigHelper(InNetworkTweetProduction.deciderByParam, deciderGateBuilder)
|
||||
val doubleDeciderOverrides: Seq[OptionalOverride[Double]] =
|
||||
configHelper.createDeciderBasedOverrides(InNetworkTweetProduction.doubleParams)
|
||||
val booleanDeciderOverrides: Seq[OptionalOverride[Boolean]] =
|
||||
configHelper.createDeciderBasedBooleanOverrides(InNetworkTweetProduction.booleanDeciderParams)
|
||||
val boundedIntFeatureSwitchOverrides: Seq[OptionalOverride[Int]] =
|
||||
FeatureSwitchOverrideUtil.getBoundedIntFSOverrides(
|
||||
InNetworkTweetProduction.boundedIntFeatureSwitchParams: _*)
|
||||
val booleanFeatureSwitchOverrides: Seq[OptionalOverride[Boolean]] =
|
||||
FeatureSwitchOverrideUtil.getBooleanFSOverrides(
|
||||
InNetworkTweetProduction.booleanFeatureSwitchParams: _*)
|
||||
|
||||
val config: BaseConfig = new BaseConfigBuilder()
|
||||
.set(
|
||||
booleanDeciderOverrides: _*
|
||||
)
|
||||
.set(
|
||||
doubleDeciderOverrides: _*
|
||||
)
|
||||
.set(
|
||||
boundedIntFeatureSwitchOverrides: _*
|
||||
)
|
||||
.set(
|
||||
booleanFeatureSwitchOverrides: _*
|
||||
)
|
||||
.build(InNetworkTweetProduction.getClass.getSimpleName)
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
|
||||
"servo/decider",
|
||||
"servo/util/src/main/scala",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,13 @@
|
||||
package com.twitter.timelineranker.parameters.monitoring
|
||||
|
||||
import com.twitter.timelines.configapi.FSParam
|
||||
|
||||
object MonitoringParams {
|
||||
|
||||
object DebugAuthorsAllowListParam
|
||||
extends FSParam[Seq[Long]](
|
||||
name = "monitoring_debug_authors_allow_list",
|
||||
default = Seq.empty[Long]
|
||||
)
|
||||
|
||||
}
|
||||
@ -0,0 +1,14 @@
|
||||
package com.twitter.timelineranker.parameters.monitoring
|
||||
|
||||
import com.twitter.timelines.configapi.BaseConfigBuilder
|
||||
import com.twitter.timelineranker.parameters.monitoring.MonitoringParams.DebugAuthorsAllowListParam
|
||||
import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil
|
||||
|
||||
object MonitoringProduction {
|
||||
private val longSeqOverrides =
|
||||
FeatureSwitchOverrideUtil.getLongSeqFSOverrides(DebugAuthorsAllowListParam)
|
||||
|
||||
val config = BaseConfigBuilder()
|
||||
.set(longSeqOverrides: _*)
|
||||
.build()
|
||||
}
|
||||
@ -0,0 +1,18 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
|
||||
"configapi/configapi-decider/src/main/scala/com/twitter/timelines/configapi/decider",
|
||||
"servo/decider",
|
||||
"servo/util/src/main/scala",
|
||||
"timelineranker/common/src/main/scala/com/twitter/timelineranker/model",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/decider",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util",
|
||||
"timelines/src/main/scala/com/twitter/timelines/experiment",
|
||||
"util/util-logging",
|
||||
"util/util-stats",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,231 @@
|
||||
package com.twitter.timelineranker.parameters.recap
|
||||
|
||||
import com.twitter.timelines.configapi.decider._
|
||||
import com.twitter.timelines.configapi.FSBoundedParam
|
||||
import com.twitter.timelines.configapi.FSParam
|
||||
import com.twitter.timelines.configapi.Param
|
||||
import com.twitter.timelines.util.bounds.BoundsWithDefault
|
||||
|
||||
object RecapParams {
|
||||
val MaxFollowedUsers: BoundsWithDefault[Int] = BoundsWithDefault[Int](1, 3000, 1000)
|
||||
val MaxCountMultiplier: BoundsWithDefault[Double] = BoundsWithDefault[Double](0.1, 2.0, 2.0)
|
||||
val MaxRealGraphAndFollowedUsers: BoundsWithDefault[Int] = BoundsWithDefault[Int](0, 2000, 1000)
|
||||
val ProbabilityRandomTweet: BoundsWithDefault[Double] = BoundsWithDefault[Double](0.0, 1.0, 0.0)
|
||||
|
||||
/**
|
||||
* Controls limit on the number of followed users fetched from SGS.
|
||||
*
|
||||
* The specific default value below is for blender-timelines parity.
|
||||
*/
|
||||
object MaxFollowedUsersParam
|
||||
extends FSBoundedParam[Int](
|
||||
name = "recap_max_followed_users",
|
||||
default = MaxFollowedUsers.default,
|
||||
min = MaxFollowedUsers.bounds.minInclusive,
|
||||
max = MaxFollowedUsers.bounds.maxInclusive
|
||||
)
|
||||
|
||||
/**
|
||||
* Controls limit on the number of hits for Earlybird.
|
||||
* We added it solely for backward compatibility, to align with recycled.
|
||||
* RecapSource is deprecated, but, this param is used by RecapAuthor source
|
||||
*/
|
||||
object RelevanceOptionsMaxHitsToProcessParam
|
||||
extends FSBoundedParam[Int](
|
||||
name = "recap_relevance_options_max_hits_to_process",
|
||||
default = 500,
|
||||
min = 100,
|
||||
max = 20000
|
||||
)
|
||||
|
||||
/**
|
||||
* Enables fetching author seedset from real graph users. Only used if user follows >= 1000.
|
||||
* If true, expands author seedset with real graph users and recent followed users.
|
||||
* Otherwise, user seedset only includes followed users.
|
||||
*/
|
||||
object EnableRealGraphUsersParam extends Param(false)
|
||||
|
||||
/**
|
||||
* Only used if EnableRealGraphUsersParam is true and OnlyRealGraphUsersParam is false.
|
||||
* Maximum number of real graph users and recent followed users when mixing recent/real-graph users.
|
||||
*/
|
||||
object MaxRealGraphAndFollowedUsersParam
|
||||
extends Param(MaxRealGraphAndFollowedUsers.default)
|
||||
with DeciderValueConverter[Int] {
|
||||
override def convert: IntConverter[Int] =
|
||||
OutputBoundIntConverter(MaxRealGraphAndFollowedUsers.bounds)
|
||||
}
|
||||
|
||||
/**
|
||||
* FS-controlled param to override the MaxRealGraphAndFollowedUsersParam decider value for experiments
|
||||
*/
|
||||
object MaxRealGraphAndFollowedUsersFSOverrideParam
|
||||
extends FSBoundedParam[Option[Int]](
|
||||
name = "max_real_graph_and_followers_users_fs_override_param",
|
||||
default = None,
|
||||
min = Some(100),
|
||||
max = Some(10000)
|
||||
)
|
||||
|
||||
/**
|
||||
* Experimental params for leveling the playing field between user folowees received from
|
||||
* real-graph and follow-graph stores.
|
||||
* Author relevance scores returned by real-graph are currently being used for light-ranking
|
||||
* in-network tweet candidates.
|
||||
* Follow-graph store returns the most recent followees without any relevance scores
|
||||
* We are trying to impute the missing scores by using aggregated statistics (min, avg, p50, etc.)
|
||||
* of real-graph scores.
|
||||
*/
|
||||
object ImputeRealGraphAuthorWeightsParam
|
||||
extends FSParam(name = "impute_real_graph_author_weights", default = false)
|
||||
|
||||
object ImputeRealGraphAuthorWeightsPercentileParam
|
||||
extends FSBoundedParam[Int](
|
||||
name = "impute_real_graph_author_weights_percentile",
|
||||
default = 50,
|
||||
min = 0,
|
||||
max = 99)
|
||||
|
||||
/**
|
||||
* Enable running the new pipeline for recap author source
|
||||
*/
|
||||
object EnableNewRecapAuthorPipeline extends Param(false)
|
||||
|
||||
/**
|
||||
* Fallback value for maximum number of search results, if not specified by query.maxCount
|
||||
*/
|
||||
object DefaultMaxTweetCount extends Param(200)
|
||||
|
||||
/**
|
||||
* We multiply maxCount (caller supplied value) by this multiplier and fetch those many
|
||||
* candidates from search so that we are left with sufficient number of candidates after
|
||||
* hydration and filtering.
|
||||
*/
|
||||
object MaxCountMultiplierParam
|
||||
extends Param(MaxCountMultiplier.default)
|
||||
with DeciderValueConverter[Double] {
|
||||
override def convert: IntConverter[Double] =
|
||||
OutputBoundIntConverter[Double](divideDeciderBy100 _, MaxCountMultiplier.bounds)
|
||||
}
|
||||
|
||||
/**
|
||||
* Enables return all results from search index.
|
||||
*/
|
||||
object EnableReturnAllResultsParam
|
||||
extends FSParam(name = "recap_enable_return_all_results", default = false)
|
||||
|
||||
/**
|
||||
* Includes one or multiple random tweets in the response.
|
||||
*/
|
||||
object IncludeRandomTweetParam
|
||||
extends FSParam(name = "recap_include_random_tweet", default = false)
|
||||
|
||||
/**
|
||||
* One single random tweet (true) or tag tweet as random with given probability (false).
|
||||
*/
|
||||
object IncludeSingleRandomTweetParam
|
||||
extends FSParam(name = "recap_include_random_tweet_single", default = true)
|
||||
|
||||
/**
|
||||
* Probability to tag a tweet as random (will not be ranked).
|
||||
*/
|
||||
object ProbabilityRandomTweetParam
|
||||
extends FSBoundedParam(
|
||||
name = "recap_include_random_tweet_probability",
|
||||
default = ProbabilityRandomTweet.default,
|
||||
min = ProbabilityRandomTweet.bounds.minInclusive,
|
||||
max = ProbabilityRandomTweet.bounds.maxInclusive)
|
||||
|
||||
/**
|
||||
* Enable extra sorting by score for search results.
|
||||
*/
|
||||
object EnableExtraSortingInSearchResultParam extends Param(true)
|
||||
|
||||
/**
|
||||
* Enables semantic core, penguin, and tweetypie content features in recap source.
|
||||
*/
|
||||
object EnableContentFeaturesHydrationParam extends Param(true)
|
||||
|
||||
/**
|
||||
* additionally enables tokens when hydrating content features.
|
||||
*/
|
||||
object EnableTokensInContentFeaturesHydrationParam
|
||||
extends FSParam(
|
||||
name = "recap_enable_tokens_in_content_features_hydration",
|
||||
default = false
|
||||
)
|
||||
|
||||
/**
|
||||
* additionally enables tweet text when hydrating content features.
|
||||
* This only works if EnableContentFeaturesHydrationParam is set to true
|
||||
*/
|
||||
object EnableTweetTextInContentFeaturesHydrationParam
|
||||
extends FSParam(
|
||||
name = "recap_enable_tweet_text_in_content_features_hydration",
|
||||
default = false
|
||||
)
|
||||
|
||||
/**
|
||||
* Enables hydrating in-network inReplyToTweet features
|
||||
*/
|
||||
object EnableInNetworkInReplyToTweetFeaturesHydrationParam
|
||||
extends FSParam(
|
||||
name = "recap_enable_in_network_in_reply_to_tweet_features_hydration",
|
||||
default = false
|
||||
)
|
||||
|
||||
/**
|
||||
* Enables hydrating root tweet of in-network replies and extended replies
|
||||
*/
|
||||
object EnableReplyRootTweetHydrationParam
|
||||
extends FSParam(
|
||||
name = "recap_enable_reply_root_tweet_hydration",
|
||||
default = false
|
||||
)
|
||||
|
||||
/**
|
||||
* Enable setting tweetTypes in search queries with TweetKindOption in RecapQuery
|
||||
*/
|
||||
object EnableSettingTweetTypesWithTweetKindOption
|
||||
extends FSParam(
|
||||
name = "recap_enable_setting_tweet_types_with_tweet_kind_option",
|
||||
default = false
|
||||
)
|
||||
|
||||
/**
|
||||
* Enable relevance search, otherwise recency search from earlybird.
|
||||
*/
|
||||
object EnableRelevanceSearchParam
|
||||
extends FSParam(
|
||||
name = "recap_enable_relevance_search",
|
||||
default = true
|
||||
)
|
||||
|
||||
object EnableExpandedExtendedRepliesFilterParam
|
||||
extends FSParam(
|
||||
name = "recap_enable_expanded_extended_replies_filter",
|
||||
default = false
|
||||
)
|
||||
|
||||
/**
|
||||
* additionally enables conversationControl when hydrating content features.
|
||||
* This only works if EnableContentFeaturesHydrationParam is set to true
|
||||
*/
|
||||
object EnableConversationControlInContentFeaturesHydrationParam
|
||||
extends FSParam(
|
||||
name = "conversation_control_in_content_features_hydration_recap_enable",
|
||||
default = false
|
||||
)
|
||||
|
||||
object EnableTweetMediaHydrationParam
|
||||
extends FSParam(
|
||||
name = "tweet_media_hydration_recap_enable",
|
||||
default = false
|
||||
)
|
||||
|
||||
object EnableExcludeSourceTweetIdsQueryParam
|
||||
extends FSParam[Boolean](
|
||||
name = "recap_exclude_source_tweet_ids_query_enable",
|
||||
default = false
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,115 @@
|
||||
package com.twitter.timelineranker.parameters.recap
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.servo.decider.DeciderGateBuilder
|
||||
import com.twitter.servo.decider.DeciderKeyName
|
||||
import com.twitter.timelineranker.decider.DeciderKey
|
||||
import com.twitter.timelineranker.parameters.recap.RecapParams._
|
||||
import com.twitter.timelineranker.parameters.util.ConfigHelper
|
||||
import com.twitter.timelines.configapi._
|
||||
import com.twitter.servo.decider.DeciderKeyEnum
|
||||
|
||||
object RecapProduction {
|
||||
val deciderByParam: Map[Param[_], DeciderKeyEnum#Value] = Map[Param[_], DeciderKeyName](
|
||||
EnableRealGraphUsersParam -> DeciderKey.EnableRealGraphUsers,
|
||||
MaxRealGraphAndFollowedUsersParam -> DeciderKey.MaxRealGraphAndFollowedUsers,
|
||||
EnableContentFeaturesHydrationParam -> DeciderKey.RecapEnableContentFeaturesHydration,
|
||||
MaxCountMultiplierParam -> DeciderKey.RecapMaxCountMultiplier,
|
||||
EnableNewRecapAuthorPipeline -> DeciderKey.RecapAuthorEnableNewPipeline,
|
||||
RecapParams.EnableExtraSortingInSearchResultParam -> DeciderKey.RecapEnableExtraSortingInResults
|
||||
)
|
||||
|
||||
val intParams: Seq[MaxRealGraphAndFollowedUsersParam.type] = Seq(
|
||||
MaxRealGraphAndFollowedUsersParam
|
||||
)
|
||||
|
||||
val doubleParams: Seq[MaxCountMultiplierParam.type] = Seq(
|
||||
MaxCountMultiplierParam
|
||||
)
|
||||
|
||||
val boundedDoubleFeatureSwitchParams: Seq[FSBoundedParam[Double]] = Seq(
|
||||
RecapParams.ProbabilityRandomTweetParam
|
||||
)
|
||||
|
||||
val booleanParams: Seq[Param[Boolean]] = Seq(
|
||||
EnableRealGraphUsersParam,
|
||||
EnableContentFeaturesHydrationParam,
|
||||
EnableNewRecapAuthorPipeline,
|
||||
RecapParams.EnableExtraSortingInSearchResultParam
|
||||
)
|
||||
|
||||
val booleanFeatureSwitchParams: Seq[FSParam[Boolean]] = Seq(
|
||||
RecapParams.EnableReturnAllResultsParam,
|
||||
RecapParams.IncludeRandomTweetParam,
|
||||
RecapParams.IncludeSingleRandomTweetParam,
|
||||
RecapParams.EnableInNetworkInReplyToTweetFeaturesHydrationParam,
|
||||
RecapParams.EnableReplyRootTweetHydrationParam,
|
||||
RecapParams.EnableSettingTweetTypesWithTweetKindOption,
|
||||
RecapParams.EnableRelevanceSearchParam,
|
||||
EnableTokensInContentFeaturesHydrationParam,
|
||||
EnableTweetTextInContentFeaturesHydrationParam,
|
||||
EnableExpandedExtendedRepliesFilterParam,
|
||||
EnableConversationControlInContentFeaturesHydrationParam,
|
||||
EnableTweetMediaHydrationParam,
|
||||
ImputeRealGraphAuthorWeightsParam,
|
||||
EnableExcludeSourceTweetIdsQueryParam
|
||||
)
|
||||
|
||||
val boundedIntFeatureSwitchParams: Seq[FSBoundedParam[Int]] = Seq(
|
||||
RecapParams.MaxFollowedUsersParam,
|
||||
ImputeRealGraphAuthorWeightsPercentileParam,
|
||||
RecapParams.RelevanceOptionsMaxHitsToProcessParam
|
||||
)
|
||||
}
|
||||
|
||||
class RecapProduction(deciderGateBuilder: DeciderGateBuilder, statsReceiver: StatsReceiver) {
|
||||
|
||||
val configHelper: ConfigHelper =
|
||||
new ConfigHelper(RecapProduction.deciderByParam, deciderGateBuilder)
|
||||
val intOverrides: Seq[OptionalOverride[Int]] =
|
||||
configHelper.createDeciderBasedOverrides(RecapProduction.intParams)
|
||||
val optionalBoundedIntFeatureSwitchOverrides: Seq[OptionalOverride[Option[Int]]] =
|
||||
FeatureSwitchOverrideUtil.getBoundedOptionalIntOverrides(
|
||||
(
|
||||
MaxRealGraphAndFollowedUsersFSOverrideParam,
|
||||
"max_real_graph_and_followers_users_fs_override_defined",
|
||||
"max_real_graph_and_followers_users_fs_override_value"
|
||||
)
|
||||
)
|
||||
val doubleOverrides: Seq[OptionalOverride[Double]] =
|
||||
configHelper.createDeciderBasedOverrides(RecapProduction.doubleParams)
|
||||
val booleanOverrides: Seq[OptionalOverride[Boolean]] =
|
||||
configHelper.createDeciderBasedBooleanOverrides(RecapProduction.booleanParams)
|
||||
val booleanFeatureSwitchOverrides: Seq[OptionalOverride[Boolean]] =
|
||||
FeatureSwitchOverrideUtil.getBooleanFSOverrides(RecapProduction.booleanFeatureSwitchParams: _*)
|
||||
val boundedDoubleFeatureSwitchOverrides: Seq[OptionalOverride[Double]] =
|
||||
FeatureSwitchOverrideUtil.getBoundedDoubleFSOverrides(
|
||||
RecapProduction.boundedDoubleFeatureSwitchParams: _*)
|
||||
val boundedIntFeatureSwitchOverrides: Seq[OptionalOverride[Int]] =
|
||||
FeatureSwitchOverrideUtil.getBoundedIntFSOverrides(
|
||||
RecapProduction.boundedIntFeatureSwitchParams: _*)
|
||||
|
||||
val config: BaseConfig = new BaseConfigBuilder()
|
||||
.set(
|
||||
intOverrides: _*
|
||||
)
|
||||
.set(
|
||||
booleanOverrides: _*
|
||||
)
|
||||
.set(
|
||||
doubleOverrides: _*
|
||||
)
|
||||
.set(
|
||||
booleanFeatureSwitchOverrides: _*
|
||||
)
|
||||
.set(
|
||||
boundedIntFeatureSwitchOverrides: _*
|
||||
)
|
||||
.set(
|
||||
optionalBoundedIntFeatureSwitchOverrides: _*
|
||||
)
|
||||
.set(
|
||||
boundedDoubleFeatureSwitchOverrides: _*
|
||||
)
|
||||
.build(RecapProduction.getClass.getSimpleName)
|
||||
}
|
||||
@ -0,0 +1,79 @@
|
||||
package com.twitter.timelineranker.parameters.recap
|
||||
|
||||
import com.twitter.timelineranker.model.RecapQuery
|
||||
import com.twitter.timelines.util.bounds.BoundsWithDefault
|
||||
|
||||
object RecapQueryContext {
|
||||
val MaxFollowedUsers: BoundsWithDefault[Int] = BoundsWithDefault[Int](1, 3000, 1000)
|
||||
val MaxCountMultiplier: BoundsWithDefault[Double] = BoundsWithDefault[Double](0.1, 2.0, 2.0)
|
||||
val MaxRealGraphAndFollowedUsers: BoundsWithDefault[Int] = BoundsWithDefault[Int](0, 2000, 1000)
|
||||
|
||||
def getDefaultContext(query: RecapQuery): RecapQueryContext = {
|
||||
new RecapQueryContextImpl(
|
||||
query,
|
||||
getEnableHydrationUsingTweetyPie = () => false,
|
||||
getMaxFollowedUsers = () => MaxFollowedUsers.default,
|
||||
getMaxCountMultiplier = () => MaxCountMultiplier.default,
|
||||
getEnableRealGraphUsers = () => false,
|
||||
getOnlyRealGraphUsers = () => false,
|
||||
getMaxRealGraphAndFollowedUsers = () => MaxRealGraphAndFollowedUsers.default,
|
||||
getEnableTextFeatureHydration = () => false
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Note that methods that return parameter value always use () to indicate that
|
||||
// side effects may be involved in their invocation.
|
||||
trait RecapQueryContext {
|
||||
def query: RecapQuery
|
||||
|
||||
// If true, tweet hydration are performed by calling TweetyPie.
|
||||
// Otherwise, tweets are partially hydrated based on information in ThriftSearchResult.
|
||||
def enableHydrationUsingTweetyPie(): Boolean
|
||||
|
||||
// Maximum number of followed user accounts to use when fetching recap tweets.
|
||||
def maxFollowedUsers(): Int
|
||||
|
||||
// We multiply maxCount (caller supplied value) by this multiplier and fetch those many
|
||||
// candidates from search so that we are left with sufficient number of candidates after
|
||||
// hydration and filtering.
|
||||
def maxCountMultiplier(): Double
|
||||
|
||||
// Only used if user follows >= 1000.
|
||||
// If true, fetches recap/recycled tweets using author seedset mixing with real graph users and followed users.
|
||||
// Otherwise, fetches recap/recycled tweets only using followed users
|
||||
def enableRealGraphUsers(): Boolean
|
||||
|
||||
// Only used if enableRealGraphUsers is true.
|
||||
// If true, user seedset only contains real graph users.
|
||||
// Otherwise, user seedset contains real graph users and recent followed users.
|
||||
def onlyRealGraphUsers(): Boolean
|
||||
|
||||
// Only used if enableRealGraphUsers is true and onlyRealGraphUsers is false.
|
||||
// Maximum number of real graph users and recent followed users when mixing recent/real-graph users.
|
||||
def maxRealGraphAndFollowedUsers(): Int
|
||||
|
||||
// If true, text features are hydrated for prediction.
|
||||
// Otherwise those feature values are not set at all.
|
||||
def enableTextFeatureHydration(): Boolean
|
||||
}
|
||||
|
||||
class RecapQueryContextImpl(
|
||||
override val query: RecapQuery,
|
||||
getEnableHydrationUsingTweetyPie: () => Boolean,
|
||||
getMaxFollowedUsers: () => Int,
|
||||
getMaxCountMultiplier: () => Double,
|
||||
getEnableRealGraphUsers: () => Boolean,
|
||||
getOnlyRealGraphUsers: () => Boolean,
|
||||
getMaxRealGraphAndFollowedUsers: () => Int,
|
||||
getEnableTextFeatureHydration: () => Boolean)
|
||||
extends RecapQueryContext {
|
||||
|
||||
override def enableHydrationUsingTweetyPie(): Boolean = { getEnableHydrationUsingTweetyPie() }
|
||||
override def maxFollowedUsers(): Int = { getMaxFollowedUsers() }
|
||||
override def maxCountMultiplier(): Double = { getMaxCountMultiplier() }
|
||||
override def enableRealGraphUsers(): Boolean = { getEnableRealGraphUsers() }
|
||||
override def onlyRealGraphUsers(): Boolean = { getOnlyRealGraphUsers() }
|
||||
override def maxRealGraphAndFollowedUsers(): Int = { getMaxRealGraphAndFollowedUsers() }
|
||||
override def enableTextFeatureHydration(): Boolean = { getEnableTextFeatureHydration() }
|
||||
}
|
||||
@ -0,0 +1,12 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
|
||||
"servo/decider",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/decider",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,53 @@
|
||||
package com.twitter.timelineranker.parameters.recap_author
|
||||
|
||||
import com.twitter.timelines.configapi.FSParam
|
||||
import com.twitter.timelines.configapi.Param
|
||||
|
||||
object RecapAuthorParams {
|
||||
|
||||
/**
|
||||
* Enables semantic core, penguin, and tweetypie content features in recap author source.
|
||||
*/
|
||||
object EnableContentFeaturesHydrationParam extends Param(false)
|
||||
|
||||
/**
|
||||
* additionally enables tokens when hydrating content features.
|
||||
*/
|
||||
object EnableTokensInContentFeaturesHydrationParam
|
||||
extends FSParam(
|
||||
name = "recap_author_enable_tokens_in_content_features_hydration",
|
||||
default = false
|
||||
)
|
||||
|
||||
/**
|
||||
* additionally enables tweet text when hydrating content features.
|
||||
* This only works if EnableContentFeaturesHydrationParam is set to true
|
||||
*/
|
||||
object EnableTweetTextInContentFeaturesHydrationParam
|
||||
extends FSParam(
|
||||
name = "recap_author_enable_tweet_text_in_content_features_hydration",
|
||||
default = false
|
||||
)
|
||||
|
||||
object EnableEarlybirdRealtimeCgMigrationParam
|
||||
extends FSParam(
|
||||
name = "recap_author_enable_earlybird_realtime_cg_migration",
|
||||
default = false
|
||||
)
|
||||
/**
|
||||
* additionally enables conversationControl when hydrating content features.
|
||||
* This only works if EnableContentFeaturesHydrationParam is set to true
|
||||
*/
|
||||
object EnableConversationControlInContentFeaturesHydrationParam
|
||||
extends FSParam(
|
||||
name = "conversation_control_in_content_features_hydration_recap_author_enable",
|
||||
default = false
|
||||
)
|
||||
|
||||
object EnableTweetMediaHydrationParam
|
||||
extends FSParam(
|
||||
name = "tweet_media_hydration_recap_author_enable",
|
||||
default = false
|
||||
)
|
||||
|
||||
}
|
||||
@ -0,0 +1,46 @@
|
||||
package com.twitter.timelineranker.parameters.recap_author
|
||||
|
||||
import com.twitter.servo.decider.DeciderGateBuilder
|
||||
import com.twitter.servo.decider.DeciderKeyName
|
||||
import com.twitter.timelineranker.decider.DeciderKey
|
||||
import com.twitter.timelineranker.parameters.recap_author.RecapAuthorParams._
|
||||
import com.twitter.timelineranker.parameters.util.ConfigHelper
|
||||
import com.twitter.timelines.configapi._
|
||||
|
||||
object RecapAuthorProduction {
|
||||
val deciderByParam: Map[Param[_], DeciderKeyName] = Map[Param[_], DeciderKeyName](
|
||||
EnableContentFeaturesHydrationParam -> DeciderKey.RecapAuthorEnableContentFeaturesHydration
|
||||
)
|
||||
|
||||
val booleanParams: Seq[EnableContentFeaturesHydrationParam.type] = Seq(
|
||||
EnableContentFeaturesHydrationParam
|
||||
)
|
||||
|
||||
val booleanFeatureSwitchParams: Seq[FSParam[Boolean]] = Seq(
|
||||
EnableTokensInContentFeaturesHydrationParam,
|
||||
EnableTweetTextInContentFeaturesHydrationParam,
|
||||
EnableConversationControlInContentFeaturesHydrationParam,
|
||||
EnableTweetMediaHydrationParam,
|
||||
EnableEarlybirdRealtimeCgMigrationParam
|
||||
)
|
||||
}
|
||||
|
||||
class RecapAuthorProduction(deciderGateBuilder: DeciderGateBuilder) {
|
||||
val configHelper: ConfigHelper =
|
||||
new ConfigHelper(RecapAuthorProduction.deciderByParam, deciderGateBuilder)
|
||||
val booleanOverrides: Seq[OptionalOverride[Boolean]] =
|
||||
configHelper.createDeciderBasedBooleanOverrides(RecapAuthorProduction.booleanParams)
|
||||
|
||||
val booleanFeatureSwitchOverrides: Seq[OptionalOverride[Boolean]] =
|
||||
FeatureSwitchOverrideUtil.getBooleanFSOverrides(
|
||||
RecapAuthorProduction.booleanFeatureSwitchParams: _*
|
||||
)
|
||||
|
||||
val config: BaseConfig = new BaseConfigBuilder()
|
||||
.set(
|
||||
booleanOverrides: _*
|
||||
).set(
|
||||
booleanFeatureSwitchOverrides: _*
|
||||
)
|
||||
.build(RecapAuthorProduction.getClass.getSimpleName)
|
||||
}
|
||||
@ -0,0 +1,12 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
|
||||
"servo/decider",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/decider",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,48 @@
|
||||
package com.twitter.timelineranker.parameters.recap_hydration
|
||||
|
||||
import com.twitter.timelines.configapi.FSParam
|
||||
import com.twitter.timelines.configapi.Param
|
||||
|
||||
object RecapHydrationParams {
|
||||
|
||||
/**
|
||||
* Enables semantic core, penguin, and tweetypie content features in recap hydration source.
|
||||
*/
|
||||
object EnableContentFeaturesHydrationParam extends Param(false)
|
||||
|
||||
/**
|
||||
* additionally enables tokens when hydrating content features.
|
||||
*/
|
||||
object EnableTokensInContentFeaturesHydrationParam
|
||||
extends FSParam(
|
||||
name = "recap_hydration_enable_tokens_in_content_features_hydration",
|
||||
default = false
|
||||
)
|
||||
|
||||
/**
|
||||
* additionally enables tweet text when hydrating content features.
|
||||
* This only works if EnableContentFeaturesHydrationParam is set to true
|
||||
*/
|
||||
object EnableTweetTextInContentFeaturesHydrationParam
|
||||
extends FSParam(
|
||||
name = "recap_hydration_enable_tweet_text_in_content_features_hydration",
|
||||
default = false
|
||||
)
|
||||
|
||||
/**
|
||||
* additionally enables conversationControl when hydrating content features.
|
||||
* This only works if EnableContentFeaturesHydrationParam is set to true
|
||||
*/
|
||||
object EnableConversationControlInContentFeaturesHydrationParam
|
||||
extends FSParam(
|
||||
name = "conversation_control_in_content_features_hydration_recap_hydration_enable",
|
||||
default = false
|
||||
)
|
||||
|
||||
object EnableTweetMediaHydrationParam
|
||||
extends FSParam(
|
||||
name = "tweet_media_hydration_recap_hydration_enable",
|
||||
default = false
|
||||
)
|
||||
|
||||
}
|
||||
@ -0,0 +1,45 @@
|
||||
package com.twitter.timelineranker.parameters.recap_hydration
|
||||
|
||||
import com.twitter.servo.decider.DeciderGateBuilder
|
||||
import com.twitter.servo.decider.DeciderKeyName
|
||||
import com.twitter.timelineranker.decider.DeciderKey
|
||||
import com.twitter.timelineranker.parameters.recap_hydration.RecapHydrationParams._
|
||||
import com.twitter.timelineranker.parameters.util.ConfigHelper
|
||||
import com.twitter.timelines.configapi._
|
||||
|
||||
object RecapHydrationProduction {
|
||||
val deciderByParam: Map[Param[_], DeciderKeyName] = Map[Param[_], DeciderKeyName](
|
||||
EnableContentFeaturesHydrationParam -> DeciderKey.RecapHydrationEnableContentFeaturesHydration
|
||||
)
|
||||
|
||||
val booleanParams: Seq[EnableContentFeaturesHydrationParam.type] = Seq(
|
||||
EnableContentFeaturesHydrationParam
|
||||
)
|
||||
|
||||
val booleanFeatureSwitchParams: Seq[FSParam[Boolean]] = Seq(
|
||||
EnableTokensInContentFeaturesHydrationParam,
|
||||
EnableTweetTextInContentFeaturesHydrationParam,
|
||||
EnableConversationControlInContentFeaturesHydrationParam,
|
||||
EnableTweetMediaHydrationParam
|
||||
)
|
||||
}
|
||||
|
||||
class RecapHydrationProduction(deciderGateBuilder: DeciderGateBuilder) {
|
||||
val configHelper: ConfigHelper =
|
||||
new ConfigHelper(RecapHydrationProduction.deciderByParam, deciderGateBuilder)
|
||||
val booleanOverrides: Seq[OptionalOverride[Boolean]] =
|
||||
configHelper.createDeciderBasedBooleanOverrides(RecapHydrationProduction.booleanParams)
|
||||
|
||||
val booleanFeatureSwitchOverrides: Seq[OptionalOverride[Boolean]] =
|
||||
FeatureSwitchOverrideUtil.getBooleanFSOverrides(
|
||||
RecapHydrationProduction.booleanFeatureSwitchParams: _*
|
||||
)
|
||||
|
||||
val config: BaseConfig = new BaseConfigBuilder()
|
||||
.set(
|
||||
booleanOverrides: _*
|
||||
).set(
|
||||
booleanFeatureSwitchOverrides: _*
|
||||
)
|
||||
.build(RecapHydrationProduction.getClass.getSimpleName)
|
||||
}
|
||||
@ -0,0 +1,18 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"configapi/configapi-core",
|
||||
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
|
||||
"servo/decider",
|
||||
"timelineranker/common/src/main/scala/com/twitter/timelineranker/model",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/config",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/decider",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util",
|
||||
"timelines/src/main/scala/com/twitter/timelines/config",
|
||||
"timelines/src/main/scala/com/twitter/timelines/decider",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util/bounds",
|
||||
"util/util-stats/src/main/scala",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,45 @@
|
||||
package com.twitter.timelineranker.parameters.revchron
|
||||
|
||||
import com.twitter.timelines.configapi.FSBoundedParam
|
||||
import com.twitter.timelines.configapi.FSParam
|
||||
|
||||
object ReverseChronParams {
|
||||
import ReverseChronTimelineQueryContext._
|
||||
|
||||
/**
|
||||
* Controls limit on the number of followed users fetched from SGS when materializing home timelines.
|
||||
*/
|
||||
object MaxFollowedUsersParam
|
||||
extends FSBoundedParam(
|
||||
"reverse_chron_max_followed_users",
|
||||
default = MaxFollowedUsers.default,
|
||||
min = MaxFollowedUsers.bounds.minInclusive,
|
||||
max = MaxFollowedUsers.bounds.maxInclusive
|
||||
)
|
||||
|
||||
object ReturnEmptyWhenOverMaxFollowsParam
|
||||
extends FSParam(
|
||||
name = "reverse_chron_return_empty_when_over_max_follows",
|
||||
default = true
|
||||
)
|
||||
|
||||
/**
|
||||
* When true, search requests for the reverse chron timeline will include an additional operator
|
||||
* so that search will not return tweets that are directed at non-followed users.
|
||||
*/
|
||||
object DirectedAtNarrowcastingViaSearchParam
|
||||
extends FSParam(
|
||||
name = "reverse_chron_directed_at_narrowcasting_via_search",
|
||||
default = false
|
||||
)
|
||||
|
||||
/**
|
||||
* When true, search requests for the reverse chron timeline will request additional metadata
|
||||
* from search and use this metadata for post filtering.
|
||||
*/
|
||||
object PostFilteringBasedOnSearchMetadataEnabledParam
|
||||
extends FSParam(
|
||||
name = "reverse_chron_post_filtering_based_on_search_metadata_enabled",
|
||||
default = true
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,28 @@
|
||||
package com.twitter.timelineranker.parameters.revchron
|
||||
|
||||
import com.twitter.servo.decider.DeciderGateBuilder
|
||||
import com.twitter.timelines.configapi._
|
||||
|
||||
object ReverseChronProduction {
|
||||
val intFeatureSwitchParams = Seq(ReverseChronParams.MaxFollowedUsersParam)
|
||||
val booleanFeatureSwitchParams = Seq(
|
||||
ReverseChronParams.ReturnEmptyWhenOverMaxFollowsParam,
|
||||
ReverseChronParams.DirectedAtNarrowcastingViaSearchParam,
|
||||
ReverseChronParams.PostFilteringBasedOnSearchMetadataEnabledParam
|
||||
)
|
||||
}
|
||||
|
||||
class ReverseChronProduction(deciderGateBuilder: DeciderGateBuilder) {
|
||||
val intOverrides = FeatureSwitchOverrideUtil.getBoundedIntFSOverrides(
|
||||
ReverseChronProduction.intFeatureSwitchParams: _*
|
||||
)
|
||||
|
||||
val booleanOverrides = FeatureSwitchOverrideUtil.getBooleanFSOverrides(
|
||||
ReverseChronProduction.booleanFeatureSwitchParams: _*
|
||||
)
|
||||
|
||||
val config: BaseConfig = new BaseConfigBuilder()
|
||||
.set(intOverrides: _*)
|
||||
.set(booleanOverrides: _*)
|
||||
.build(ReverseChronProduction.getClass.getSimpleName)
|
||||
}
|
||||
@ -0,0 +1,114 @@
|
||||
package com.twitter.timelineranker.parameters.revchron
|
||||
|
||||
import com.twitter.timelineranker.model.ReverseChronTimelineQuery
|
||||
import com.twitter.timelines.util.bounds.BoundsWithDefault
|
||||
import com.twitter.timelineservice.model.core.TimelineKind
|
||||
import com.twitter.timelineservice.model.core.TimelineLimits
|
||||
|
||||
object ReverseChronTimelineQueryContext {
|
||||
val MaxCountLimit: Int = TimelineLimits.default.lengthLimit(TimelineKind.home)
|
||||
val MaxCount: BoundsWithDefault[Int] = BoundsWithDefault[Int](0, MaxCountLimit, MaxCountLimit)
|
||||
val MaxCountMultiplier: BoundsWithDefault[Double] = BoundsWithDefault[Double](0.5, 2.0, 1.0)
|
||||
val MaxFollowedUsers: BoundsWithDefault[Int] = BoundsWithDefault[Int](1, 15000, 5000)
|
||||
val TweetsFilteringLossageThresholdPercent: BoundsWithDefault[Int] =
|
||||
BoundsWithDefault[Int](10, 100, 20)
|
||||
val TweetsFilteringLossageLimitPercent: BoundsWithDefault[Int] =
|
||||
BoundsWithDefault[Int](40, 65, 60)
|
||||
|
||||
def getDefaultContext(query: ReverseChronTimelineQuery): ReverseChronTimelineQueryContext = {
|
||||
new ReverseChronTimelineQueryContextImpl(
|
||||
query,
|
||||
getMaxCount = () => MaxCount.default,
|
||||
getMaxCountMultiplier = () => MaxCountMultiplier.default,
|
||||
getMaxFollowedUsers = () => MaxFollowedUsers.default,
|
||||
getReturnEmptyWhenOverMaxFollows = () => true,
|
||||
getDirectedAtNarrowastingViaSearch = () => false,
|
||||
getPostFilteringBasedOnSearchMetadataEnabled = () => true,
|
||||
getBackfillFilteredEntries = () => false,
|
||||
getTweetsFilteringLossageThresholdPercent = () =>
|
||||
TweetsFilteringLossageThresholdPercent.default,
|
||||
getTweetsFilteringLossageLimitPercent = () => TweetsFilteringLossageLimitPercent.default
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Note that methods that return parameter value always use () to indicate that
|
||||
// side effects may be involved in their invocation.
|
||||
// for example, A likely side effect is to cause experiment impression.
|
||||
trait ReverseChronTimelineQueryContext {
|
||||
def query: ReverseChronTimelineQuery
|
||||
|
||||
// Maximum number of tweets to be returned to caller.
|
||||
def maxCount(): Int
|
||||
|
||||
// Multiplier applied to the number of tweets fetched from search expressed as percentage.
|
||||
// It can be used to fetch more than the number tweets requested by a caller (to improve similarity)
|
||||
// or to fetch less than requested to reduce load.
|
||||
def maxCountMultiplier(): Double
|
||||
|
||||
// Maximum number of followed user accounts to use when materializing home timelines.
|
||||
def maxFollowedUsers(): Int
|
||||
|
||||
// When true, if the user follows more than maxFollowedUsers, return an empty timeline.
|
||||
def returnEmptyWhenOverMaxFollows(): Boolean
|
||||
|
||||
// When true, appends an operator for directed-at narrowcasting to the home materialization
|
||||
// search request
|
||||
def directedAtNarrowcastingViaSearch(): Boolean
|
||||
|
||||
// When true, requests additional metadata from search and use this metadata for post filtering.
|
||||
def postFilteringBasedOnSearchMetadataEnabled(): Boolean
|
||||
|
||||
// Controls whether to back-fill timeline entries that get filtered out by TweetsPostFilter
|
||||
// during home timeline materialization.
|
||||
def backfillFilteredEntries(): Boolean
|
||||
|
||||
// If back-filling filtered entries is enabled and if number of tweets that get filtered out
|
||||
// exceed this percentage then we will issue a second call to get more tweets.
|
||||
def tweetsFilteringLossageThresholdPercent(): Int
|
||||
|
||||
// We need to ensure that the number of tweets requested by the second call
|
||||
// are not unbounded (for example, if everything is filtered out in the first call)
|
||||
// therefore we adjust the actual filtered out percentage to be no greater than
|
||||
// the value below.
|
||||
def tweetsFilteringLossageLimitPercent(): Int
|
||||
|
||||
// We need to indicate to search if we should use the archive cluster
|
||||
// this option will come from ReverseChronTimelineQueryOptions and
|
||||
// will be `true` by default if the options are not present.
|
||||
def getTweetsFromArchiveIndex(): Boolean =
|
||||
query.options.map(_.getTweetsFromArchiveIndex).getOrElse(true)
|
||||
}
|
||||
|
||||
class ReverseChronTimelineQueryContextImpl(
|
||||
override val query: ReverseChronTimelineQuery,
|
||||
getMaxCount: () => Int,
|
||||
getMaxCountMultiplier: () => Double,
|
||||
getMaxFollowedUsers: () => Int,
|
||||
getReturnEmptyWhenOverMaxFollows: () => Boolean,
|
||||
getDirectedAtNarrowastingViaSearch: () => Boolean,
|
||||
getPostFilteringBasedOnSearchMetadataEnabled: () => Boolean,
|
||||
getBackfillFilteredEntries: () => Boolean,
|
||||
getTweetsFilteringLossageThresholdPercent: () => Int,
|
||||
getTweetsFilteringLossageLimitPercent: () => Int)
|
||||
extends ReverseChronTimelineQueryContext {
|
||||
override def maxCount(): Int = { getMaxCount() }
|
||||
override def maxCountMultiplier(): Double = { getMaxCountMultiplier() }
|
||||
override def maxFollowedUsers(): Int = { getMaxFollowedUsers() }
|
||||
override def backfillFilteredEntries(): Boolean = { getBackfillFilteredEntries() }
|
||||
override def tweetsFilteringLossageThresholdPercent(): Int = {
|
||||
getTweetsFilteringLossageThresholdPercent()
|
||||
}
|
||||
override def tweetsFilteringLossageLimitPercent(): Int = {
|
||||
getTweetsFilteringLossageLimitPercent()
|
||||
}
|
||||
override def returnEmptyWhenOverMaxFollows(): Boolean = {
|
||||
getReturnEmptyWhenOverMaxFollows()
|
||||
}
|
||||
override def directedAtNarrowcastingViaSearch(): Boolean = {
|
||||
getDirectedAtNarrowastingViaSearch()
|
||||
}
|
||||
override def postFilteringBasedOnSearchMetadataEnabled(): Boolean = {
|
||||
getPostFilteringBasedOnSearchMetadataEnabled()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,72 @@
|
||||
package com.twitter.timelineranker.parameters.revchron
|
||||
|
||||
import com.twitter.timelineranker.config.RuntimeConfiguration
|
||||
import com.twitter.timelineranker.decider.DeciderKey
|
||||
import com.twitter.timelineranker.model._
|
||||
import com.twitter.timelineranker.parameters.util.RequestContextBuilder
|
||||
import com.twitter.timelines.configapi.Config
|
||||
import com.twitter.timelines.decider.FeatureValue
|
||||
import com.twitter.util.Future
|
||||
|
||||
object ReverseChronTimelineQueryContextBuilder {
|
||||
val MaxCountLimitKey: Seq[String] = Seq("search_request_max_count_limit")
|
||||
}
|
||||
|
||||
class ReverseChronTimelineQueryContextBuilder(
|
||||
config: Config,
|
||||
runtimeConfig: RuntimeConfiguration,
|
||||
requestContextBuilder: RequestContextBuilder) {
|
||||
|
||||
import ReverseChronTimelineQueryContext._
|
||||
import ReverseChronTimelineQueryContextBuilder._
|
||||
|
||||
private val maxCountMultiplier = FeatureValue(
|
||||
runtimeConfig.deciderGateBuilder,
|
||||
DeciderKey.MultiplierOfMaterializationTweetsFetched,
|
||||
MaxCountMultiplier,
|
||||
value => (value / 100.0)
|
||||
)
|
||||
|
||||
private val backfillFilteredEntriesGate =
|
||||
runtimeConfig.deciderGateBuilder.linearGate(DeciderKey.BackfillFilteredEntries)
|
||||
|
||||
private val tweetsFilteringLossageThresholdPercent = FeatureValue(
|
||||
runtimeConfig.deciderGateBuilder,
|
||||
DeciderKey.TweetsFilteringLossageThreshold,
|
||||
TweetsFilteringLossageThresholdPercent,
|
||||
value => (value / 100)
|
||||
)
|
||||
|
||||
private val tweetsFilteringLossageLimitPercent = FeatureValue(
|
||||
runtimeConfig.deciderGateBuilder,
|
||||
DeciderKey.TweetsFilteringLossageLimit,
|
||||
TweetsFilteringLossageLimitPercent,
|
||||
value => (value / 100)
|
||||
)
|
||||
|
||||
private def getMaxCountFromConfigStore(): Int = {
|
||||
runtimeConfig.configStore.getAsInt(MaxCountLimitKey).getOrElse(MaxCount.default)
|
||||
}
|
||||
|
||||
def apply(query: ReverseChronTimelineQuery): Future[ReverseChronTimelineQueryContext] = {
|
||||
requestContextBuilder(Some(query.userId), deviceContext = None).map { baseContext =>
|
||||
val params = config(baseContext, runtimeConfig.statsReceiver)
|
||||
|
||||
new ReverseChronTimelineQueryContextImpl(
|
||||
query,
|
||||
getMaxCount = () => getMaxCountFromConfigStore(),
|
||||
getMaxCountMultiplier = () => maxCountMultiplier(),
|
||||
getMaxFollowedUsers = () => params(ReverseChronParams.MaxFollowedUsersParam),
|
||||
getReturnEmptyWhenOverMaxFollows =
|
||||
() => params(ReverseChronParams.ReturnEmptyWhenOverMaxFollowsParam),
|
||||
getDirectedAtNarrowastingViaSearch =
|
||||
() => params(ReverseChronParams.DirectedAtNarrowcastingViaSearchParam),
|
||||
getPostFilteringBasedOnSearchMetadataEnabled =
|
||||
() => params(ReverseChronParams.PostFilteringBasedOnSearchMetadataEnabledParam),
|
||||
getBackfillFilteredEntries = () => backfillFilteredEntriesGate(),
|
||||
getTweetsFilteringLossageThresholdPercent = () => tweetsFilteringLossageThresholdPercent(),
|
||||
getTweetsFilteringLossageLimitPercent = () => tweetsFilteringLossageLimitPercent()
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,13 @@
|
||||
scala_library(
|
||||
sources = ["*.scala"],
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
strict_deps = True,
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
|
||||
"servo/decider/src/main/scala",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/decider",
|
||||
"timelineranker/server/src/main/scala/com/twitter/timelineranker/parameters/util",
|
||||
"timelines/src/main/scala/com/twitter/timelines/util/bounds",
|
||||
],
|
||||
)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user