mirror of
https://github.com/twitter/the-algorithm.git
synced 2025-06-12 15:48:19 -05:00
Twitter Recommendation Algorithm
Please note we have force-pushed a new initial commit in order to remove some publicly-available Twitter user information. Note that this process may be required in the future.
This commit is contained in:
twitter-team
@ -0,0 +1,20 @@
|
||||
resources(
|
||||
sources = [
|
||||
"*.tsv",
|
||||
"*.xml",
|
||||
"**/*",
|
||||
"config/*.yml",
|
||||
],
|
||||
)
|
||||
|
||||
# Created for Bazel compatibility.
|
||||
# In Bazel, loose files must be part of a target to be included into a bundle.
|
||||
files(
|
||||
name = "frs_resources",
|
||||
sources = [
|
||||
"*.tsv",
|
||||
"*.xml",
|
||||
"*.yml",
|
||||
"**/*",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,129 @@
|
||||
enable_recommendations:
|
||||
comment: Proportion of requests where we return an actual response as part. Decreasing the value will increase the portion of empty responses (in order to disable the service) as part of the graceful degradation.
|
||||
default_availability: 10000
|
||||
enable_score_user_candidates:
|
||||
comment: Proportion of requests where score user candidates from the scoreUserCandidates endpoint
|
||||
default_availability: 10000
|
||||
enable_profile_sidebar_product:
|
||||
comment: Proportion of requests where we return an actual response for profile sidebar product
|
||||
default_availability: 10000
|
||||
enable_magic_recs_product:
|
||||
comment: Proportion of requests where we return an actual response for magic recs product
|
||||
default_availability: 10000
|
||||
enable_rux_landing_page_product:
|
||||
comment: Proportion of requests where we return an actual response for rux landing page product
|
||||
default_availability: 10000
|
||||
enable_rux_pymk_product:
|
||||
comment: Proportion of requests where we return an actual response for rux pymk product
|
||||
default_availability: 10000
|
||||
enable_profile_bonus_follow_product:
|
||||
comment: Proportion of requests where we return an actual response for profile bonus follow product
|
||||
default_availability: 10000
|
||||
enable_election_explore_wtf_product:
|
||||
comment: Proportion of requests where we return an actual response for election explore wtf product
|
||||
default_availability: 10000
|
||||
enable_cluster_follow_product:
|
||||
comment: Proportion of requests where we return an actual response for cluster follow product
|
||||
default_availability: 10000
|
||||
enable_home_timeline_product:
|
||||
comment: Proportion of requests where we return an actual response for htl wtf product
|
||||
default_availability: 10000
|
||||
enable_htl_bonus_follow_product:
|
||||
comment: Proportion of requests where we return an actual response for htl bonus follow product
|
||||
default_availability: 10000
|
||||
enable_explore_tab_product:
|
||||
comment: Proportion of requests where we return an actual response for explore tab product
|
||||
default_availability: 10000
|
||||
enable_sidebar_product:
|
||||
comment: Proportion of requests where we return an actual response for sidebar product
|
||||
default_availability: 10000
|
||||
enable_campaign_form_product:
|
||||
comment: Proportion of requests where we return an actual response for campaign form product
|
||||
default_availability: 10000
|
||||
enable_reactive_follow_product:
|
||||
comment: Proportion of requests where we return an actual response for reactive follow product
|
||||
default_availability: 10000
|
||||
enable_nux_pymk_product:
|
||||
comment: Proportion of requests where we return an actual response for nux pymk product
|
||||
default_availability: 10000
|
||||
enable_nux_interests_product:
|
||||
comment: Proportion of requests where we return an actual response for nux interests product
|
||||
default_availability: 10000
|
||||
enable_nux_topic_bonus_follow_product:
|
||||
comment: Proportion of requests where we return an actual response for nux topic-based bonus follow product
|
||||
default_availability: 10000
|
||||
enable_india_covid19_curated_accounts_wtf_product:
|
||||
comment: Proportion of requests where we return an actual response for india covid19 curated accounts wtf product
|
||||
default_availability: 10000
|
||||
enable_ab_upload_product:
|
||||
comment: Proportion of requests where we return an actual response for the address book upload product
|
||||
default_availability: 10000
|
||||
enable_people_plus_plus_product:
|
||||
comment: Proportion of requests where we return an actual response for the PeoplePlusPlus/Connect Tab product
|
||||
default_availability: 10000
|
||||
enable_tweet_notification_recs_product:
|
||||
comment: Proportion of requests where we return an actual response for the Tweet Notification Recommendations product
|
||||
default_availability: 10000
|
||||
enable_profile_device_follow_product:
|
||||
comment: Proportion of requests where we return an actual response for the ProfileDeviceFollow product
|
||||
default_availability: 10000
|
||||
enable_diffy_module_dark_reading:
|
||||
comment: Percentage of dark read traffic routed to diffy thrift
|
||||
default_availability: 0
|
||||
enable_recos_backfill_product:
|
||||
comment: Proportion of requests where we return an actual response for the RecosBackfill product
|
||||
default_availability: 10000
|
||||
enable_post_nux_follow_task_product:
|
||||
comment: Proportion of requests where we return an actual response for post NUX follow task product
|
||||
default_availability: 10000
|
||||
enable_curated_space_hosts_product:
|
||||
comment: Proportion of requests where we return an actual response for curated space hosts product
|
||||
default_availability: 10000
|
||||
enable_nux_geo_category_product:
|
||||
comment: Proportion of requests where we return an actual response for nux geo category product
|
||||
default_availability: 10000
|
||||
enable_nux_interests_category_product:
|
||||
comment: Proportion of requests where we return an actual response for nux interests category product
|
||||
default_availability: 10000
|
||||
enable_nux_pymk_category_product:
|
||||
comment: Proportion of requests where we return an actual response for nux pymk category product
|
||||
default_availability: 10000
|
||||
enable_home_timeline_tweet_recs_product:
|
||||
comment: Proportion of requests where we return an actual response for the Home Timeline Tweet Recs product
|
||||
default_availability: 10000
|
||||
enable_htl_bulk_friend_follows_product:
|
||||
comment: Proportion of requests where we return an actual response for the HTL bulk friend follows product
|
||||
default_availability: 10000
|
||||
enable_nux_auto_follow_product:
|
||||
comment: Proportion of requests where we return an actual response for the NUX auto follow product
|
||||
default_availability: 10000
|
||||
enable_search_bonus_follow_product:
|
||||
comment: Proportion of requests where we return an actual response for search bonus follow product
|
||||
default_availability: 10000
|
||||
enable_fetch_user_in_request_builder:
|
||||
comment: Proportion of requests where we fetch user object from gizmoduck in request builder
|
||||
default_availability: 0
|
||||
enable_product_mixer_magic_recs_product:
|
||||
comment: Proportion of requests where we enable the product mixer magic recs product
|
||||
default_availability: 10000
|
||||
enable_home_timeline_reverse_chron_product:
|
||||
comment: Proportion of requests where we return an actual response for Home timeline reverse chron product
|
||||
default_availability: 10000
|
||||
enable_product_mixer_pipeline_magic_recs_dark_read:
|
||||
comment: Compare product mixer pipeline responses to current FRS pipeline responses for Magic Recs
|
||||
default_availability: 0
|
||||
enable_experimental_caching:
|
||||
comment: Proportion of requests we use experimental caching for data caching
|
||||
default_availability: 0
|
||||
enable_distributed_caching:
|
||||
comment: Proportion of requests we use a distributed cache cluster for data caching
|
||||
default_availability: 10000
|
||||
enable_gizmoduck_caching:
|
||||
comment: Proportion of requests we use a distributed cache cluster for data caching in Gizmoduck
|
||||
default_availability: 10000
|
||||
enable_traffic_dark_reading:
|
||||
comment: Proportion of requests where we replicate the request for traffic dark reading
|
||||
default_availability: 0
|
||||
enable_graph_feature_service_requests:
|
||||
comment: Proportion of requests where we allow request calls to Graph Feature Service
|
||||
default_availability: 10000
|
||||
@ -0,0 +1,133 @@
|
||||
<configuration>
|
||||
<shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
|
||||
|
||||
<!-- ===================================================== -->
|
||||
<!-- Service Config -->
|
||||
<!-- ===================================================== -->
|
||||
<property name="DEFAULT_SERVICE_PATTERN"
|
||||
value="%-16X{traceId} %-12X{serviceIdentifier:--} %-16X{method} %-12X{product:--} %-25logger{0} %msg"/>
|
||||
|
||||
<property name="DEFAULT_ACCESS_PATTERN"
|
||||
value="%msg %-12X{serviceIdentifier:--} %X{traceId} %X{product:--}"/>
|
||||
|
||||
<!-- ===================================================== -->
|
||||
<!-- Common Config -->
|
||||
<!-- ===================================================== -->
|
||||
|
||||
<!-- JUL/JDK14 to Logback bridge -->
|
||||
<contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
|
||||
<resetJUL>true</resetJUL>
|
||||
</contextListener>
|
||||
|
||||
<!-- Service Log (Rollover every 50MB, max 5 logs) -->
|
||||
<appender name="SERVICE" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${log.service.output}</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
|
||||
<fileNamePattern>${log.service.output}.%i</fileNamePattern>
|
||||
<minIndex>1</minIndex>
|
||||
<maxIndex>5</maxIndex>
|
||||
</rollingPolicy>
|
||||
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
|
||||
<maxFileSize>50MB</maxFileSize>
|
||||
</triggeringPolicy>
|
||||
<encoder>
|
||||
<pattern>%date %.-3level ${DEFAULT_SERVICE_PATTERN}%n</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<!-- Access Log (Rollover every 50MB, max 5 logs) -->
|
||||
<appender name="ACCESS" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${log.access.output}</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
|
||||
<fileNamePattern>${log.access.output}.%i</fileNamePattern>
|
||||
<minIndex>1</minIndex>
|
||||
<maxIndex>5</maxIndex>
|
||||
</rollingPolicy>
|
||||
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
|
||||
<maxFileSize>50MB</maxFileSize>
|
||||
</triggeringPolicy>
|
||||
<encoder>
|
||||
<pattern>${DEFAULT_ACCESS_PATTERN}%n</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<!--LogLens -->
|
||||
<appender name="LOGLENS" class="com.twitter.loglens.logback.LoglensAppender">
|
||||
<mdcAdditionalContext>true</mdcAdditionalContext>
|
||||
<category>${log.lens.category}</category>
|
||||
<index>${log.lens.index}</index>
|
||||
<tag>${log.lens.tag}/service</tag>
|
||||
<encoder>
|
||||
<pattern>%msg</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<!-- LogLens Access -->
|
||||
<appender name="LOGLENS-ACCESS" class="com.twitter.loglens.logback.LoglensAppender">
|
||||
<mdcAdditionalContext>true</mdcAdditionalContext>
|
||||
<category>${log.lens.category}</category>
|
||||
<index>${log.lens.index}</index>
|
||||
<tag>${log.lens.tag}/access</tag>
|
||||
<encoder>
|
||||
<pattern>%msg</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<!-- ===================================================== -->
|
||||
<!-- Primary Async Appenders -->
|
||||
<!-- ===================================================== -->
|
||||
|
||||
<property name="async_queue_size" value="${queue.size:-50000}"/>
|
||||
<property name="async_max_flush_time" value="${max.flush.time:-0}"/>
|
||||
|
||||
<appender name="ASYNC-SERVICE" class="com.twitter.inject.logback.AsyncAppender">
|
||||
<queueSize>${async_queue_size}</queueSize>
|
||||
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
|
||||
<appender-ref ref="SERVICE"/>
|
||||
</appender>
|
||||
|
||||
<appender name="ASYNC-ACCESS" class="com.twitter.inject.logback.AsyncAppender">
|
||||
<queueSize>${async_queue_size}</queueSize>
|
||||
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
|
||||
<appender-ref ref="ACCESS"/>
|
||||
</appender>
|
||||
|
||||
<appender name="ASYNC-LOGLENS" class="com.twitter.inject.logback.AsyncAppender">
|
||||
<queueSize>${async_queue_size}</queueSize>
|
||||
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
|
||||
<appender-ref ref="LOGLENS"/>
|
||||
</appender>
|
||||
|
||||
<appender name="ASYNC-LOGLENS-ACCESS" class="com.twitter.inject.logback.AsyncAppender">
|
||||
<queueSize>${async_queue_size}</queueSize>
|
||||
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
|
||||
<appender-ref ref="LOGLENS-ACCESS"/>
|
||||
</appender>
|
||||
|
||||
<!-- ===================================================== -->
|
||||
<!-- Package Config -->
|
||||
<!-- ===================================================== -->
|
||||
|
||||
<!-- Per-Package Config -->
|
||||
<logger name="com.twitter" level="info"/>
|
||||
<logger name="com.twitter.wilyns" level="warn"/>
|
||||
<logger name="com.twitter.finagle.mux" level="warn"/>
|
||||
<logger name="com.twitter.finagle.serverset2" level="warn"/>
|
||||
<logger name="com.twitter.logging.ScribeHandler" level="warn"/>
|
||||
<logger name="com.twitter.zookeeper.client.internal" level="warn"/>
|
||||
|
||||
<!-- Root Config -->
|
||||
<root level="${log_level:-INFO}">
|
||||
<appender-ref ref="ASYNC-SERVICE"/>
|
||||
<appender-ref ref="ASYNC-LOGLENS"/>
|
||||
</root>
|
||||
|
||||
<!-- Access Logging -->
|
||||
<logger name="com.twitter.finatra.thrift.filters.AccessLoggingFilter"
|
||||
level="info"
|
||||
additivity="false">
|
||||
<appender-ref ref="ASYNC-ACCESS"/>
|
||||
<appender-ref ref="ASYNC-LOGLENS-ACCESS"/>
|
||||
</logger>
|
||||
|
||||
</configuration>
|
||||
@ -0,0 +1,8 @@
|
||||
# OWNER = jdeng
|
||||
# Date = 20141223_153423
|
||||
# Training Size = 16744473
|
||||
# Testing Size = 16767335
|
||||
# trained with ElasticNetCV alpha=0.05 cv_folds=5 best_lambda=1.0E-7
|
||||
# num base features: 10
|
||||
# num nonzero weights: 30
|
||||
{bias:-5.67151,featureMetadataMap:["fwd_email":{metadata:{featureWeight:{weight:2.47389}}},"rev_phone":{metadata:{featureWeight:{weight:1.88433}}},"mutual_follow_path":{metadata:{featureWeight:{intervalWeights:[{left:47.0,weight:6.31809},{left:11.0,right:16.0,weight:4.52959},{left:31.0,right:47.0,weight:5.7101},{right:2.0,weight:0.383515},{left:24.0,right:31.0,weight:5.26515},{left:3.0,right:4.0,weight:2.91751},{left:2.0,right:3.0,weight:2.22851},{left:4.0,right:5.0,weight:3.28515},{left:8.0,right:11.0,weight:4.14731},{left:5.0,right:8.0,weight:3.73588},{left:16.0,right:24.0,weight:4.90908}]}}},"fwd_phone":{metadata:{featureWeight:{weight:2.07327}}},"fwd_email_path":{metadata:{featureWeight:{weight:0.961773}}},"rev_phone_path":{metadata:{featureWeight:{weight:0.354484}}},"low_tweepcred_follow_path":{metadata:{featureWeight:{intervalWeights:[{left:4.0,right:5.0,weight:0.177209},{left:7.0,right:8.0,weight:0.12378},{left:3.0,right:4.0,weight:0.197566},{left:5.0,right:6.0,weight:0.15867},{left:2.0,right:3.0,weight:0.196539},{right:2.0,weight:0.1805},{left:75.0,weight:-0.424598},{left:6.0,right:7.0,weight:0.143698},{left:10.0,right:13.0,weight:0.0458502},{left:8.0,right:10.0,weight:0.0919314},{left:13.0,right:75.0,weight:-0.111484}]}}},"rev_email_path":{metadata:{featureWeight:{weight:0.654451}}},"rev_email":{metadata:{featureWeight:{weight:2.33859}}},"fwd_phone_path":{metadata:{featureWeight:{weight:0.210418}}}]}
|
||||
@ -0,0 +1 @@
|
||||
{input:{context:"discover.prod",startDateTime:"",endDateTime:"",trainingFeatures:["STP_FEATURES":["fwd_email","mutual_follow_path","fwd_email_path","rev_phone_path","low_tweepcred_follow_path","rev_phone","fwd_phone","rev_email_path","rev_email","fwd_phone_path"]],engagementActions:["click","favorite","open_link","open","send_tweet","send_reply","retweet","reply","profile_click","follow"],impressionActions:["discard","results","impression"],dataFormat:1,dataPath:"",isLabeled:0},sample:{positiveSampleRatio:1.0,negativeSampleRatio:1.0,sampleType:1},split:{trainingDataSplitSize:0.5,testingDataSplitSize:0.5,splitType:2},transform:{},filter:{featureOptions:[]},join:{engagementRules:["discover"],contentIdType:"tweet",groupBucketSize:3600000},discretize:{}}
|
||||
@ -0,0 +1,48 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"finagle/finagle-http/src/main/scala",
|
||||
"finagle/finagle-thriftmux/src/main/scala",
|
||||
"finatra-internal/decider/src/main/scala",
|
||||
"finatra-internal/international/src/main/scala/com/twitter/finatra/international/modules",
|
||||
"finatra-internal/mtls-http/src/main/scala",
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/http-core/src/main/java/com/twitter/finatra/http",
|
||||
"finatra/inject/inject-app/src/main/scala",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"finatra/inject/inject-server/src/main/scala",
|
||||
"finatra/inject/inject-thrift-client",
|
||||
"finatra/jackson/src/main/scala/com/twitter/finatra/jackson/modules",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/addressbook",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/adserver",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/cache",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/email_storage_service",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/gizmoduck",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/graph_feature_service",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/phone_storage_service",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/socialgraph",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/strato",
|
||||
"follow-recommendations-service/server/src/main/resources",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/controllers",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/modules",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services/exceptions",
|
||||
"follow-recommendations-service/thrift/src/main/thrift:thrift-scala",
|
||||
"geoduck/service/src/main/scala/com/twitter/geoduck/service/common/clientmodules",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/controllers",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/functional_component/configapi",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/module/stringcenter",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice",
|
||||
"twitter-server/server/src/main/scala",
|
||||
"util/util-app/src/main/scala",
|
||||
"util/util-core:scala",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,118 @@
|
||||
package com.twitter.follow_recommendations
|
||||
|
||||
import com.google.inject.Module
|
||||
import com.twitter.finagle.ThriftMux
|
||||
import com.twitter.finatra.decider.modules.DeciderModule
|
||||
import com.twitter.finatra.http.HttpServer
|
||||
import com.twitter.finatra.http.routing.HttpRouter
|
||||
import com.twitter.finatra.international.modules.I18nFactoryModule
|
||||
import com.twitter.finatra.international.modules.LanguagesModule
|
||||
import com.twitter.finatra.jackson.modules.ScalaObjectMapperModule
|
||||
import com.twitter.finatra.mtls.http.{Mtls => HttpMtls}
|
||||
import com.twitter.finatra.mtls.thriftmux.Mtls
|
||||
import com.twitter.finatra.thrift.ThriftServer
|
||||
import com.twitter.finatra.thrift.filters._
|
||||
import com.twitter.finagle.thrift.Protocols
|
||||
import com.twitter.finatra.thrift.routing.ThriftRouter
|
||||
import com.twitter.follow_recommendations.common.clients.addressbook.AddressbookModule
|
||||
import com.twitter.follow_recommendations.common.clients.adserver.AdserverModule
|
||||
import com.twitter.follow_recommendations.common.clients.cache.MemcacheModule
|
||||
import com.twitter.follow_recommendations.common.clients.deepbirdv2.DeepBirdV2PredictionServiceClientModule
|
||||
import com.twitter.follow_recommendations.common.clients.email_storage_service.EmailStorageServiceModule
|
||||
import com.twitter.follow_recommendations.common.clients.geoduck.LocationServiceModule
|
||||
import com.twitter.follow_recommendations.common.clients.gizmoduck.GizmoduckModule
|
||||
import com.twitter.follow_recommendations.common.clients.graph_feature_service.GraphFeatureStoreModule
|
||||
import com.twitter.follow_recommendations.common.clients.impression_store.ImpressionStoreModule
|
||||
import com.twitter.follow_recommendations.common.clients.phone_storage_service.PhoneStorageServiceModule
|
||||
import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphModule
|
||||
import com.twitter.follow_recommendations.common.clients.strato.StratoClientModule
|
||||
import com.twitter.follow_recommendations.common.constants.ServiceConstants._
|
||||
import com.twitter.follow_recommendations.common.feature_hydration.sources.HydrationSourcesModule
|
||||
import com.twitter.follow_recommendations.controllers.ThriftController
|
||||
import com.twitter.follow_recommendations.modules._
|
||||
import com.twitter.follow_recommendations.service.exceptions.UnknownLoggingExceptionMapper
|
||||
import com.twitter.follow_recommendations.services.FollowRecommendationsServiceWarmupHandler
|
||||
import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService
|
||||
import com.twitter.geoduck.service.common.clientmodules.ReverseGeocoderThriftClientModule
|
||||
import com.twitter.inject.thrift.filters.DarkTrafficFilter
|
||||
import com.twitter.inject.thrift.modules.ThriftClientIdModule
|
||||
import com.twitter.product_mixer.core.controllers.ProductMixerController
|
||||
import com.twitter.product_mixer.core.module.PipelineExecutionLoggerModule
|
||||
import com.twitter.product_mixer.core.module.product_mixer_flags.ProductMixerFlagModule
|
||||
import com.twitter.product_mixer.core.module.stringcenter.ProductScopeStringCenterModule
|
||||
import com.twitter.product_mixer.core.product.guice.ProductScopeModule
|
||||
|
||||
object FollowRecommendationsServiceThriftServerMain extends FollowRecommendationsServiceThriftServer
|
||||
|
||||
class FollowRecommendationsServiceThriftServer
|
||||
extends ThriftServer
|
||||
with Mtls
|
||||
with HttpServer
|
||||
with HttpMtls {
|
||||
override val name: String = "follow-recommendations-service-server"
|
||||
|
||||
override val modules: Seq[Module] =
|
||||
Seq(
|
||||
ABDeciderModule,
|
||||
AddressbookModule,
|
||||
AdserverModule,
|
||||
ConfigApiModule,
|
||||
DeciderModule,
|
||||
DeepBirdV2PredictionServiceClientModule,
|
||||
DiffyModule,
|
||||
EmailStorageServiceModule,
|
||||
FeaturesSwitchesModule,
|
||||
FlagsModule,
|
||||
GizmoduckModule,
|
||||
GraphFeatureStoreModule,
|
||||
HydrationSourcesModule,
|
||||
I18nFactoryModule,
|
||||
ImpressionStoreModule,
|
||||
LanguagesModule,
|
||||
LocationServiceModule,
|
||||
MemcacheModule,
|
||||
PhoneStorageServiceModule,
|
||||
PipelineExecutionLoggerModule,
|
||||
ProductMixerFlagModule,
|
||||
ProductRegistryModule,
|
||||
new ProductScopeModule(),
|
||||
new ProductScopeStringCenterModule(),
|
||||
new ReverseGeocoderThriftClientModule,
|
||||
ScalaObjectMapperModule,
|
||||
ScorerModule,
|
||||
ScribeModule,
|
||||
SocialGraphModule,
|
||||
StratoClientModule,
|
||||
ThriftClientIdModule,
|
||||
TimerModule,
|
||||
)
|
||||
|
||||
def configureThrift(router: ThriftRouter): Unit = {
|
||||
router
|
||||
.filter[LoggingMDCFilter]
|
||||
.filter[TraceIdMDCFilter]
|
||||
.filter[ThriftMDCFilter]
|
||||
.filter[StatsFilter]
|
||||
.filter[AccessLoggingFilter]
|
||||
.filter[ExceptionMappingFilter]
|
||||
.exceptionMapper[UnknownLoggingExceptionMapper]
|
||||
.filter[DarkTrafficFilter[FollowRecommendationsThriftService.ReqRepServicePerEndpoint]]
|
||||
.add[ThriftController]
|
||||
}
|
||||
|
||||
override def configureThriftServer(server: ThriftMux.Server): ThriftMux.Server = {
|
||||
server.withProtocolFactory(
|
||||
Protocols.binaryFactory(
|
||||
stringLengthLimit = StringLengthLimit,
|
||||
containerLengthLimit = ContainerLengthLimit))
|
||||
}
|
||||
|
||||
override def configureHttp(router: HttpRouter): Unit = router.add(
|
||||
ProductMixerController[FollowRecommendationsThriftService.MethodPerEndpoint](
|
||||
this.injector,
|
||||
FollowRecommendationsThriftService.ExecutePipeline))
|
||||
|
||||
override def warmup(): Unit = {
|
||||
handle[FollowRecommendationsServiceWarmupHandler]()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,9 @@
|
||||
package com.twitter.follow_recommendations.assembler.models
|
||||
|
||||
import com.twitter.follow_recommendations.{thriftscala => t}
|
||||
|
||||
case class Action(text: String, actionURL: String) {
|
||||
lazy val toThrift: t.Action = {
|
||||
t.Action(text, actionURL)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,12 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"follow-recommendations-service/thrift/src/main/thrift:thrift-scala",
|
||||
"stringcenter/client",
|
||||
],
|
||||
exports = [
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,8 @@
|
||||
package com.twitter.follow_recommendations.assembler.models
|
||||
|
||||
import com.twitter.stringcenter.client.core.ExternalString
|
||||
|
||||
case class HeaderConfig(title: TitleConfig)
|
||||
case class TitleConfig(text: ExternalString)
|
||||
case class FooterConfig(actionConfig: Option[ActionConfig])
|
||||
case class ActionConfig(footerText: ExternalString, actionURL: String)
|
||||
@ -0,0 +1,13 @@
|
||||
package com.twitter.follow_recommendations.assembler.models
|
||||
|
||||
import com.twitter.follow_recommendations.{thriftscala => t}
|
||||
|
||||
trait FeedbackAction {
|
||||
def toThrift: t.FeedbackAction
|
||||
}
|
||||
|
||||
case class DismissUserId() extends FeedbackAction {
|
||||
override lazy val toThrift: t.FeedbackAction = {
|
||||
t.FeedbackAction.DismissUserId(t.DismissUserId())
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,9 @@
|
||||
package com.twitter.follow_recommendations.assembler.models
|
||||
|
||||
import com.twitter.follow_recommendations.{thriftscala => t}
|
||||
|
||||
case class Footer(action: Option[Action]) {
|
||||
lazy val toThrift: t.Footer = {
|
||||
t.Footer(action.map(_.toThrift))
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,9 @@
|
||||
package com.twitter.follow_recommendations.assembler.models
|
||||
|
||||
import com.twitter.follow_recommendations.{thriftscala => t}
|
||||
|
||||
case class Header(title: Title) {
|
||||
lazy val toThrift: t.Header = {
|
||||
t.Header(title.toThrift)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,16 @@
|
||||
package com.twitter.follow_recommendations.assembler.models
|
||||
|
||||
sealed trait Layout
|
||||
|
||||
case class UserListLayout(
|
||||
header: Option[HeaderConfig],
|
||||
userListOptions: UserListOptions,
|
||||
socialProofs: Option[Seq[SocialProof]],
|
||||
footer: Option[FooterConfig])
|
||||
extends Layout
|
||||
|
||||
case class CarouselLayout(
|
||||
header: Option[HeaderConfig],
|
||||
carouselOptions: CarouselOptions,
|
||||
socialProofs: Option[Seq[SocialProof]])
|
||||
extends Layout
|
||||
@ -0,0 +1,11 @@
|
||||
package com.twitter.follow_recommendations.assembler.models
|
||||
|
||||
sealed trait RecommendationOptions
|
||||
|
||||
case class UserListOptions(
|
||||
userBioEnabled: Boolean,
|
||||
userBioTruncated: Boolean,
|
||||
userBioMaxLines: Option[Long],
|
||||
) extends RecommendationOptions
|
||||
|
||||
case class CarouselOptions() extends RecommendationOptions
|
||||
@ -0,0 +1,16 @@
|
||||
package com.twitter.follow_recommendations.assembler.models
|
||||
|
||||
import com.twitter.stringcenter.client.core.ExternalString
|
||||
|
||||
sealed trait SocialProof
|
||||
|
||||
case class GeoContextProof(popularInCountryText: ExternalString) extends SocialProof
|
||||
case class FollowedByUsersProof(text1: ExternalString, text2: ExternalString, textN: ExternalString)
|
||||
extends SocialProof
|
||||
|
||||
sealed trait SocialText {
|
||||
def text: String
|
||||
}
|
||||
|
||||
case class GeoSocialText(text: String) extends SocialText
|
||||
case class FollowedByUsersText(text: String) extends SocialText
|
||||
@ -0,0 +1,9 @@
|
||||
package com.twitter.follow_recommendations.assembler.models
|
||||
|
||||
import com.twitter.follow_recommendations.{thriftscala => t}
|
||||
|
||||
case class Title(text: String) {
|
||||
lazy val toThrift: t.Title = {
|
||||
t.Title(text)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,47 @@
|
||||
package com.twitter.follow_recommendations.assembler.models
|
||||
|
||||
import com.twitter.follow_recommendations.{thriftscala => t}
|
||||
|
||||
trait WTFPresentation {
|
||||
def toThrift: t.WTFPresentation
|
||||
}
|
||||
|
||||
case class UserList(
|
||||
userBioEnabled: Boolean,
|
||||
userBioTruncated: Boolean,
|
||||
userBioMaxLines: Option[Long],
|
||||
feedbackAction: Option[FeedbackAction])
|
||||
extends WTFPresentation {
|
||||
def toThrift: t.WTFPresentation = {
|
||||
t.WTFPresentation.UserBioList(
|
||||
t.UserList(userBioEnabled, userBioTruncated, userBioMaxLines, feedbackAction.map(_.toThrift)))
|
||||
}
|
||||
}
|
||||
|
||||
object UserList {
|
||||
def fromUserListOptions(
|
||||
userListOptions: UserListOptions
|
||||
): UserList = {
|
||||
UserList(
|
||||
userListOptions.userBioEnabled,
|
||||
userListOptions.userBioTruncated,
|
||||
userListOptions.userBioMaxLines,
|
||||
None)
|
||||
}
|
||||
}
|
||||
|
||||
case class Carousel(
|
||||
feedbackAction: Option[FeedbackAction])
|
||||
extends WTFPresentation {
|
||||
def toThrift: t.WTFPresentation = {
|
||||
t.WTFPresentation.Carousel(t.Carousel(feedbackAction.map(_.toThrift)))
|
||||
}
|
||||
}
|
||||
|
||||
object Carousel {
|
||||
def fromCarouselOptions(
|
||||
carouselOptions: CarouselOptions
|
||||
): Carousel = {
|
||||
Carousel(None)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,16 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/com/google/inject/extensions:guice-assistedinject",
|
||||
"3rdparty/jvm/net/codingwell:scala-guice",
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,138 @@
|
||||
package com.twitter.follow_recommendations.blenders
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.base.Transform
|
||||
import com.twitter.follow_recommendations.common.models.AdMetadata
|
||||
import com.twitter.follow_recommendations.common.models.Recommendation
|
||||
import com.twitter.inject.Logging
|
||||
import com.twitter.stitch.Stitch
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class PromotedAccountsBlender @Inject() (statsReceiver: StatsReceiver)
|
||||
extends Transform[Int, Recommendation]
|
||||
with Logging {
|
||||
|
||||
import PromotedAccountsBlender._
|
||||
val stats = statsReceiver.scope(Name)
|
||||
val inputOrganicAccounts = stats.counter(InputOrganic)
|
||||
val inputPromotedAccounts = stats.counter(InputPromoted)
|
||||
val outputOrganicAccounts = stats.counter(OutputOrganic)
|
||||
val outputPromotedAccounts = stats.counter(OutputPromoted)
|
||||
val promotedAccountsStats = stats.scope(NumPromotedAccounts)
|
||||
|
||||
override def transform(
|
||||
maxResults: Int,
|
||||
items: Seq[Recommendation]
|
||||
): Stitch[Seq[Recommendation]] = {
|
||||
val (promoted, organic) = items.partition(_.isPromotedAccount)
|
||||
val promotedIds = promoted.map(_.id).toSet
|
||||
val dedupedOrganic = organic.filterNot(u => promotedIds.contains(u.id))
|
||||
val blended = blendPromotedAccount(dedupedOrganic, promoted, maxResults)
|
||||
val (outputPromoted, outputOrganic) = blended.partition(_.isPromotedAccount)
|
||||
inputOrganicAccounts.incr(dedupedOrganic.size)
|
||||
inputPromotedAccounts.incr(promoted.size)
|
||||
outputOrganicAccounts.incr(outputOrganic.size)
|
||||
val size = outputPromoted.size
|
||||
outputPromotedAccounts.incr(size)
|
||||
if (size <= 5) {
|
||||
promotedAccountsStats.counter(outputPromoted.size.toString).incr()
|
||||
} else {
|
||||
promotedAccountsStats.counter(MoreThan5Promoted).incr()
|
||||
}
|
||||
Stitch.value(blended)
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge Promoted results and organic results. Promoted result dictates the position
|
||||
* in the merge list.
|
||||
*
|
||||
* merge a list of positioned users, aka. promoted, and a list of organic
|
||||
* users. The positioned promoted users are pre-sorted with regards to their
|
||||
* position ascendingly. Only requirement about position is to be within the
|
||||
* range, i.e, can not exceed the combined length if merge is successful, ok
|
||||
* to be at the last position, but not beyond.
|
||||
* For more detailed description of location position:
|
||||
* http://confluence.local.twitter.com/display/ADS/Promoted+Tweets+in+Timeline+Design+Document
|
||||
*/
|
||||
@VisibleForTesting
|
||||
private[blenders] def mergePromotedAccounts(
|
||||
organicUsers: Seq[Recommendation],
|
||||
promotedUsers: Seq[Recommendation]
|
||||
): Seq[Recommendation] = {
|
||||
def mergeAccountWithIndex(
|
||||
organicUsers: Seq[Recommendation],
|
||||
promotedUsers: Seq[Recommendation],
|
||||
index: Int
|
||||
): Stream[Recommendation] = {
|
||||
if (promotedUsers.isEmpty) organicUsers.toStream
|
||||
else {
|
||||
val promotedHead = promotedUsers.head
|
||||
val promotedTail = promotedUsers.tail
|
||||
promotedHead.adMetadata match {
|
||||
case Some(AdMetadata(position, _)) =>
|
||||
if (position < 0) mergeAccountWithIndex(organicUsers, promotedTail, index)
|
||||
else if (position == index)
|
||||
promotedHead #:: mergeAccountWithIndex(organicUsers, promotedTail, index)
|
||||
else if (organicUsers.isEmpty) organicUsers.toStream
|
||||
else {
|
||||
val organicHead = organicUsers.head
|
||||
val organicTail = organicUsers.tail
|
||||
organicHead #:: mergeAccountWithIndex(organicTail, promotedUsers, index + 1)
|
||||
}
|
||||
case _ =>
|
||||
logger.error("Unknown Candidate type in mergePromotedAccounts")
|
||||
Stream.empty
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mergeAccountWithIndex(organicUsers, promotedUsers, 0)
|
||||
}
|
||||
|
||||
private[this] def blendPromotedAccount(
|
||||
organic: Seq[Recommendation],
|
||||
promoted: Seq[Recommendation],
|
||||
maxResults: Int
|
||||
): Seq[Recommendation] = {
|
||||
|
||||
val merged = mergePromotedAccounts(organic, promoted)
|
||||
val mergedServed = merged.take(maxResults)
|
||||
val promotedServed = promoted.intersect(mergedServed)
|
||||
|
||||
if (isBlendPromotedNeeded(
|
||||
mergedServed.size - promotedServed.size,
|
||||
promotedServed.size,
|
||||
maxResults
|
||||
)) {
|
||||
mergedServed
|
||||
} else {
|
||||
organic.take(maxResults)
|
||||
}
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
private[blenders] def isBlendPromotedNeeded(
|
||||
organicSize: Int,
|
||||
promotedSize: Int,
|
||||
maxResults: Int
|
||||
): Boolean = {
|
||||
(organicSize > 1) &&
|
||||
(promotedSize > 0) &&
|
||||
(promotedSize < organicSize) &&
|
||||
(promotedSize <= 2) &&
|
||||
(maxResults > 1)
|
||||
}
|
||||
}
|
||||
|
||||
object PromotedAccountsBlender {
|
||||
val Name = "promoted_accounts_blender"
|
||||
val InputOrganic = "input_organic_accounts"
|
||||
val InputPromoted = "input_promoted_accounts"
|
||||
val OutputOrganic = "output_organic_accounts"
|
||||
val OutputPromoted = "output_promoted_accounts"
|
||||
val NumPromotedAccounts = "num_promoted_accounts"
|
||||
val MoreThan5Promoted = "more_than_5"
|
||||
}
|
||||
@ -0,0 +1,28 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"configapi/configapi-core",
|
||||
"configapi/configapi-decider",
|
||||
"configapi/configapi-featureswitches:v2",
|
||||
"featureswitches/featureswitches-core",
|
||||
"featureswitches/featureswitches-core:v2",
|
||||
"featureswitches/featureswitches-core/src/main/scala/com/twitter/featureswitches/v2/builder",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/flows/content_recommender_flow",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,16 @@
|
||||
package com.twitter.follow_recommendations.configapi
|
||||
|
||||
import com.twitter.timelines.configapi.CompositeConfig
|
||||
import com.twitter.timelines.configapi.Config
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class ConfigBuilder @Inject() (
|
||||
deciderConfigs: DeciderConfigs,
|
||||
featureSwitchConfigs: FeatureSwitchConfigs) {
|
||||
// The order of configs added to `CompositeConfig` is important. The config will be matched with
|
||||
// the first possible rule. So, current setup will give priority to Deciders instead of FS
|
||||
def build(): Config =
|
||||
new CompositeConfig(Seq(deciderConfigs.config, featureSwitchConfigs.config))
|
||||
}
|
||||
@ -0,0 +1,52 @@
|
||||
package com.twitter.follow_recommendations.configapi
|
||||
|
||||
import com.twitter.decider.Recipient
|
||||
import com.twitter.decider.SimpleRecipient
|
||||
import com.twitter.follow_recommendations.configapi.deciders.DeciderKey
|
||||
import com.twitter.follow_recommendations.configapi.deciders.DeciderParams
|
||||
import com.twitter.follow_recommendations.products.home_timeline_tweet_recs.configapi.HomeTimelineTweetRecsParams
|
||||
import com.twitter.servo.decider.DeciderGateBuilder
|
||||
import com.twitter.timelines.configapi._
|
||||
import com.twitter.timelines.configapi.decider.DeciderSwitchOverrideValue
|
||||
import com.twitter.timelines.configapi.decider.GuestRecipient
|
||||
import com.twitter.timelines.configapi.decider.RecipientBuilder
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class DeciderConfigs @Inject() (deciderGateBuilder: DeciderGateBuilder) {
|
||||
val overrides: Seq[OptionalOverride[_]] = DeciderConfigs.ParamsToDeciderMap.map {
|
||||
case (params, deciderKey) =>
|
||||
params.optionalOverrideValue(
|
||||
DeciderSwitchOverrideValue(
|
||||
feature = deciderGateBuilder.keyToFeature(deciderKey),
|
||||
enabledValue = true,
|
||||
recipientBuilder = DeciderConfigs.UserOrGuestOrRequest
|
||||
)
|
||||
)
|
||||
}.toSeq
|
||||
|
||||
val config: BaseConfig = BaseConfigBuilder(overrides).build("FollowRecommendationServiceDeciders")
|
||||
}
|
||||
|
||||
object DeciderConfigs {
|
||||
val ParamsToDeciderMap = Map(
|
||||
DeciderParams.EnableRecommendations -> DeciderKey.EnableRecommendations,
|
||||
DeciderParams.EnableScoreUserCandidates -> DeciderKey.EnableScoreUserCandidates,
|
||||
HomeTimelineTweetRecsParams.EnableProduct -> DeciderKey.EnableHomeTimelineTweetRecsProduct,
|
||||
)
|
||||
|
||||
object UserOrGuestOrRequest extends RecipientBuilder {
|
||||
|
||||
def apply(requestContext: BaseRequestContext): Option[Recipient] = requestContext match {
|
||||
case c: WithUserId if c.userId.isDefined =>
|
||||
c.userId.map(SimpleRecipient)
|
||||
case c: WithGuestId if c.guestId.isDefined =>
|
||||
c.guestId.map(GuestRecipient)
|
||||
case c: WithGuestId =>
|
||||
RecipientBuilder.Request(c)
|
||||
case _ =>
|
||||
throw new UndefinedUserIdNorGuestIDException(requestContext)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,138 @@
|
||||
package com.twitter.follow_recommendations.configapi
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.base.SocialProofEnforcedCandidateSourceFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeoQualityFollowSourceFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeoSourceFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSourceFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.real_graph.RealGraphOonFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RepeatedProfileVisitsFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.sims.SimsSourceFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.SimsExpansionFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.socialgraph.RecentFollowingRecentFollowingExpansionSourceFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStpSourceFsConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.stp.OnlineSTPSourceFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsFSConfig
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.user_user_graph.UserUserGraphFSConfig
|
||||
import com.twitter.follow_recommendations.common.feature_hydration.sources.FeatureHydrationSourcesFSConfig
|
||||
import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.WeightedCandidateSourceRankerFSConfig
|
||||
import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig
|
||||
import com.twitter.follow_recommendations.flows.content_recommender_flow.ContentRecommenderFlowFSConfig
|
||||
import com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicateFSConfig
|
||||
import com.twitter.follow_recommendations.common.predicates.hss.HssPredicateFSConfig
|
||||
import com.twitter.follow_recommendations.common.predicates.sgs.SgsPredicateFSConfig
|
||||
import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlFlowFSConfig
|
||||
import com.twitter.logging.Logger
|
||||
import com.twitter.timelines.configapi.BaseConfigBuilder
|
||||
import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil
|
||||
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class FeatureSwitchConfigs @Inject() (
|
||||
globalFeatureSwitchConfig: GlobalFeatureSwitchConfig,
|
||||
featureHydrationSourcesFSConfig: FeatureHydrationSourcesFSConfig,
|
||||
weightedCandidateSourceRankerFSConfig: WeightedCandidateSourceRankerFSConfig,
|
||||
// Flow related config
|
||||
contentRecommenderFlowFSConfig: ContentRecommenderFlowFSConfig,
|
||||
postNuxMlFlowFSConfig: PostNuxMlFlowFSConfig,
|
||||
// Candidate source related config
|
||||
crowdSearchAccountsFSConfig: CrowdSearchAccountsFSConfig,
|
||||
offlineStpSourceFsConfig: OfflineStpSourceFsConfig,
|
||||
onlineSTPSourceFSConfig: OnlineSTPSourceFSConfig,
|
||||
popGeoSourceFSConfig: PopGeoSourceFSConfig,
|
||||
popGeoQualityFollowFSConfig: PopGeoQualityFollowSourceFSConfig,
|
||||
realGraphOonFSConfig: RealGraphOonFSConfig,
|
||||
repeatedProfileVisitsFSConfig: RepeatedProfileVisitsFSConfig,
|
||||
recentEngagementSimilarUsersFSConfig: RecentEngagementSimilarUsersFSConfig,
|
||||
recentFollowingRecentFollowingExpansionSourceFSConfig: RecentFollowingRecentFollowingExpansionSourceFSConfig,
|
||||
simsExpansionFSConfig: SimsExpansionFSConfig,
|
||||
simsSourceFSConfig: SimsSourceFSConfig,
|
||||
socialProofEnforcedCandidateSourceFSConfig: SocialProofEnforcedCandidateSourceFSConfig,
|
||||
triangularLoopsFSConfig: TriangularLoopsFSConfig,
|
||||
userUserGraphFSConfig: UserUserGraphFSConfig,
|
||||
// Predicate related configs
|
||||
gizmoduckPredicateFSConfig: GizmoduckPredicateFSConfig,
|
||||
hssPredicateFSConfig: HssPredicateFSConfig,
|
||||
sgsPredicateFSConfig: SgsPredicateFSConfig,
|
||||
ppmiLocaleSourceFSConfig: PPMILocaleFollowSourceFSConfig,
|
||||
topOrganicFollowsAccountsFSConfig: TopOrganicFollowsAccountsFSConfig,
|
||||
statsReceiver: StatsReceiver) {
|
||||
|
||||
val logger = Logger(classOf[FeatureSwitchConfigs])
|
||||
|
||||
val mergedFSConfig =
|
||||
FeatureSwitchConfig.merge(
|
||||
Seq(
|
||||
globalFeatureSwitchConfig,
|
||||
featureHydrationSourcesFSConfig,
|
||||
weightedCandidateSourceRankerFSConfig,
|
||||
// Flow related config
|
||||
contentRecommenderFlowFSConfig,
|
||||
postNuxMlFlowFSConfig,
|
||||
// Candidate source related config
|
||||
crowdSearchAccountsFSConfig,
|
||||
offlineStpSourceFsConfig,
|
||||
onlineSTPSourceFSConfig,
|
||||
popGeoSourceFSConfig,
|
||||
popGeoQualityFollowFSConfig,
|
||||
realGraphOonFSConfig,
|
||||
repeatedProfileVisitsFSConfig,
|
||||
recentEngagementSimilarUsersFSConfig,
|
||||
recentFollowingRecentFollowingExpansionSourceFSConfig,
|
||||
simsExpansionFSConfig,
|
||||
simsSourceFSConfig,
|
||||
socialProofEnforcedCandidateSourceFSConfig,
|
||||
triangularLoopsFSConfig,
|
||||
userUserGraphFSConfig,
|
||||
// Predicate related configs:
|
||||
gizmoduckPredicateFSConfig,
|
||||
hssPredicateFSConfig,
|
||||
sgsPredicateFSConfig,
|
||||
ppmiLocaleSourceFSConfig,
|
||||
topOrganicFollowsAccountsFSConfig,
|
||||
)
|
||||
)
|
||||
|
||||
/**
|
||||
* enum params have to be listed in this main file together as otherwise we'll have to pass in
|
||||
* some signature like `Seq[FSEnumParams[_]]` which are generics of generics and won't compile.
|
||||
* we only have enumFsParams from globalFeatureSwitchConfig at the moment
|
||||
*/
|
||||
val enumOverrides = globalFeatureSwitchConfig.enumFsParams.flatMap { enumParam =>
|
||||
FeatureSwitchOverrideUtil.getEnumFSOverrides(statsReceiver, logger, enumParam)
|
||||
}
|
||||
|
||||
val gatedOverrides = mergedFSConfig.gatedOverridesMap.flatMap {
|
||||
case (fsName, overrides) =>
|
||||
FeatureSwitchOverrideUtil.gatedOverrides(fsName, overrides: _*)
|
||||
}
|
||||
|
||||
val enumSeqOverrides = globalFeatureSwitchConfig.enumSeqFsParams.flatMap { enumSeqParam =>
|
||||
FeatureSwitchOverrideUtil.getEnumSeqFSOverrides(statsReceiver, logger, enumSeqParam)
|
||||
}
|
||||
|
||||
val overrides =
|
||||
FeatureSwitchOverrideUtil
|
||||
.getBooleanFSOverrides(mergedFSConfig.booleanFSParams: _*) ++
|
||||
FeatureSwitchOverrideUtil
|
||||
.getBoundedIntFSOverrides(mergedFSConfig.intFSParams: _*) ++
|
||||
FeatureSwitchOverrideUtil
|
||||
.getBoundedLongFSOverrides(mergedFSConfig.longFSParams: _*) ++
|
||||
FeatureSwitchOverrideUtil
|
||||
.getBoundedDoubleFSOverrides(mergedFSConfig.doubleFSParams: _*) ++
|
||||
FeatureSwitchOverrideUtil
|
||||
.getDurationFSOverrides(mergedFSConfig.durationFSParams: _*) ++
|
||||
FeatureSwitchOverrideUtil
|
||||
.getBoundedOptionalDoubleOverrides(mergedFSConfig.optionalDoubleFSParams: _*) ++
|
||||
FeatureSwitchOverrideUtil.getStringSeqFSOverrides(mergedFSConfig.stringSeqFSParams: _*) ++
|
||||
enumOverrides ++
|
||||
gatedOverrides ++
|
||||
enumSeqOverrides
|
||||
|
||||
val config = BaseConfigBuilder(overrides).build("FollowRecommendationServiceFeatureSwitches")
|
||||
}
|
||||
@ -0,0 +1,49 @@
|
||||
package com.twitter.follow_recommendations.configapi
|
||||
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsParams.AccountsFilteringAndRankingLogics
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsParams.{
|
||||
AccountsFilteringAndRankingLogics => OrganicAccountsFilteringAndRankingLogics
|
||||
}
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersParams
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.SimsExpansionSourceParams
|
||||
import com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking.MlRankerParams.CandidateScorerIdParam
|
||||
import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig
|
||||
import com.twitter.follow_recommendations.configapi.params.GlobalParams.CandidateSourcesToFilter
|
||||
import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableCandidateParamHydrations
|
||||
import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableGFSSocialProofTransform
|
||||
import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableRecommendationFlowLogs
|
||||
import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableWhoToFollowProducts
|
||||
import com.twitter.follow_recommendations.configapi.params.GlobalParams.KeepSocialUserCandidate
|
||||
import com.twitter.follow_recommendations.configapi.params.GlobalParams.KeepUserCandidate
|
||||
import com.twitter.timelines.configapi.FSName
|
||||
import com.twitter.timelines.configapi.Param
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class GlobalFeatureSwitchConfig @Inject() () extends FeatureSwitchConfig {
|
||||
override val booleanFSParams: Seq[Param[Boolean] with FSName] = {
|
||||
Seq(
|
||||
EnableCandidateParamHydrations,
|
||||
KeepUserCandidate,
|
||||
KeepSocialUserCandidate,
|
||||
EnableGFSSocialProofTransform,
|
||||
EnableWhoToFollowProducts,
|
||||
EnableRecommendationFlowLogs
|
||||
)
|
||||
}
|
||||
|
||||
val enumFsParams =
|
||||
Seq(
|
||||
CandidateScorerIdParam,
|
||||
SimsExpansionSourceParams.Aggregator,
|
||||
RecentEngagementSimilarUsersParams.Aggregator,
|
||||
CandidateSourcesToFilter,
|
||||
)
|
||||
|
||||
val enumSeqFsParams =
|
||||
Seq(
|
||||
AccountsFilteringAndRankingLogics,
|
||||
OrganicAccountsFilteringAndRankingLogics
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,29 @@
|
||||
package com.twitter.follow_recommendations.configapi
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.models.DisplayLocation
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.ClientContext
|
||||
import com.twitter.servo.util.MemoizingStatsReceiver
|
||||
import com.twitter.timelines.configapi.Config
|
||||
import com.twitter.timelines.configapi.FeatureValue
|
||||
import com.twitter.timelines.configapi.Params
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class ParamsFactory @Inject() (
|
||||
config: Config,
|
||||
requestContextFactory: RequestContextFactory,
|
||||
statsReceiver: StatsReceiver) {
|
||||
|
||||
private val stats = new MemoizingStatsReceiver(statsReceiver.scope("configapi"))
|
||||
def apply(followRecommendationServiceRequestContext: RequestContext): Params =
|
||||
config(followRecommendationServiceRequestContext, stats)
|
||||
|
||||
def apply(
|
||||
clientContext: ClientContext,
|
||||
displayLocation: DisplayLocation,
|
||||
featureOverrides: Map[String, FeatureValue]
|
||||
): Params =
|
||||
apply(requestContextFactory(clientContext, displayLocation, featureOverrides))
|
||||
}
|
||||
@ -0,0 +1,19 @@
|
||||
package com.twitter.follow_recommendations.configapi
|
||||
|
||||
import com.twitter.timelines.configapi.BaseRequestContext
|
||||
import com.twitter.timelines.configapi.FeatureContext
|
||||
import com.twitter.timelines.configapi.NullFeatureContext
|
||||
import com.twitter.timelines.configapi.GuestId
|
||||
import com.twitter.timelines.configapi.UserId
|
||||
import com.twitter.timelines.configapi.WithFeatureContext
|
||||
import com.twitter.timelines.configapi.WithGuestId
|
||||
import com.twitter.timelines.configapi.WithUserId
|
||||
|
||||
case class RequestContext(
|
||||
userId: Option[UserId],
|
||||
guestId: Option[GuestId],
|
||||
featureContext: FeatureContext = NullFeatureContext)
|
||||
extends BaseRequestContext
|
||||
with WithUserId
|
||||
with WithGuestId
|
||||
with WithFeatureContext
|
||||
@ -0,0 +1,74 @@
|
||||
package com.twitter.follow_recommendations.configapi
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting
|
||||
import com.google.inject.Inject
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.featureswitches.v2.FeatureSwitches
|
||||
import com.twitter.featureswitches.{Recipient => FeatureSwitchRecipient}
|
||||
import com.twitter.follow_recommendations.common.models.DisplayLocation
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.ClientContext
|
||||
import com.twitter.snowflake.id.SnowflakeId
|
||||
import com.twitter.timelines.configapi.FeatureContext
|
||||
import com.twitter.timelines.configapi.FeatureValue
|
||||
import com.twitter.timelines.configapi.ForcedFeatureContext
|
||||
import com.twitter.timelines.configapi.OrElseFeatureContext
|
||||
import com.twitter.timelines.configapi.featureswitches.v2.FeatureSwitchResultsFeatureContext
|
||||
import javax.inject.Singleton
|
||||
|
||||
/*
|
||||
* Request Context Factory is used to build RequestContext objects which are used
|
||||
* by the config api to determine the param overrides to apply to the request.
|
||||
* The param overrides are determined per request by configs which specify which
|
||||
* FS/Deciders/AB translate to what param overrides.
|
||||
*/
|
||||
@Singleton
|
||||
class RequestContextFactory @Inject() (featureSwitches: FeatureSwitches, decider: Decider) {
|
||||
def apply(
|
||||
clientContext: ClientContext,
|
||||
displayLocation: DisplayLocation,
|
||||
featureOverrides: Map[String, FeatureValue]
|
||||
): RequestContext = {
|
||||
val featureContext = getFeatureContext(clientContext, displayLocation, featureOverrides)
|
||||
RequestContext(clientContext.userId, clientContext.guestId, featureContext)
|
||||
}
|
||||
|
||||
private[configapi] def getFeatureContext(
|
||||
clientContext: ClientContext,
|
||||
displayLocation: DisplayLocation,
|
||||
featureOverrides: Map[String, FeatureValue]
|
||||
): FeatureContext = {
|
||||
val recipient =
|
||||
getFeatureSwitchRecipient(clientContext)
|
||||
.withCustomFields("display_location" -> displayLocation.toFsName)
|
||||
|
||||
// userAgeOpt is going to be set to None for logged out users and defaulted to Some(Int.MaxValue) for non-snowflake users
|
||||
val userAgeOpt = clientContext.userId.map { userId =>
|
||||
SnowflakeId.timeFromIdOpt(userId).map(_.untilNow.inDays).getOrElse(Int.MaxValue)
|
||||
}
|
||||
val recipientWithAccountAge =
|
||||
userAgeOpt
|
||||
.map(age => recipient.withCustomFields("account_age_in_days" -> age)).getOrElse(recipient)
|
||||
|
||||
val results = featureSwitches.matchRecipient(recipientWithAccountAge)
|
||||
OrElseFeatureContext(
|
||||
ForcedFeatureContext(featureOverrides),
|
||||
new FeatureSwitchResultsFeatureContext(results))
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
private[configapi] def getFeatureSwitchRecipient(
|
||||
clientContext: ClientContext
|
||||
): FeatureSwitchRecipient = {
|
||||
FeatureSwitchRecipient(
|
||||
userId = clientContext.userId,
|
||||
userRoles = clientContext.userRoles,
|
||||
deviceId = clientContext.deviceId,
|
||||
guestId = clientContext.guestId,
|
||||
languageCode = clientContext.languageCode,
|
||||
countryCode = clientContext.countryCode,
|
||||
isVerified = None,
|
||||
clientApplicationId = clientContext.appId,
|
||||
isTwoffice = clientContext.isTwoffice
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,18 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"configapi/configapi-core",
|
||||
"configapi/configapi-decider",
|
||||
"configapi/configapi-featureswitches:v2",
|
||||
"featureswitches/featureswitches-core",
|
||||
"featureswitches/featureswitches-core:v2",
|
||||
"featureswitches/featureswitches-core/src/main/scala/com/twitter/featureswitches/v2/builder",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,19 @@
|
||||
package com.twitter.follow_recommendations.configapi.candidates
|
||||
|
||||
import com.twitter.timelines.configapi.BaseRequestContext
|
||||
import com.twitter.timelines.configapi.FeatureContext
|
||||
import com.twitter.timelines.configapi.NullFeatureContext
|
||||
import com.twitter.timelines.configapi.WithFeatureContext
|
||||
import com.twitter.timelines.configapi.WithUserId
|
||||
|
||||
/**
|
||||
* represent the context for a recommendation candidate (producer side)
|
||||
* @param userId id of the recommended user
|
||||
* @param featureContext feature context
|
||||
*/
|
||||
case class CandidateUserContext(
|
||||
override val userId: Option[Long],
|
||||
featureContext: FeatureContext = NullFeatureContext)
|
||||
extends BaseRequestContext
|
||||
with WithUserId
|
||||
with WithFeatureContext
|
||||
@ -0,0 +1,55 @@
|
||||
package com.twitter.follow_recommendations.configapi.candidates
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting
|
||||
import com.google.inject.Inject
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.featureswitches.v2.FeatureSwitches
|
||||
import com.twitter.featureswitches.{Recipient => FeatureSwitchRecipient}
|
||||
import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants.PRODUCER_SIDE_FEATURE_SWITCHES
|
||||
import com.twitter.follow_recommendations.common.models.CandidateUser
|
||||
import com.twitter.follow_recommendations.common.models.DisplayLocation
|
||||
import com.twitter.timelines.configapi.FeatureContext
|
||||
import com.twitter.timelines.configapi.featureswitches.v2.FeatureSwitchResultsFeatureContext
|
||||
import javax.inject.Named
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class CandidateUserContextFactory @Inject() (
|
||||
@Named(PRODUCER_SIDE_FEATURE_SWITCHES) featureSwitches: FeatureSwitches,
|
||||
decider: Decider) {
|
||||
def apply(
|
||||
candidateUser: CandidateUser,
|
||||
displayLocation: DisplayLocation
|
||||
): CandidateUserContext = {
|
||||
val featureContext = getFeatureContext(candidateUser, displayLocation)
|
||||
|
||||
CandidateUserContext(Some(candidateUser.id), featureContext)
|
||||
}
|
||||
|
||||
private[configapi] def getFeatureContext(
|
||||
candidateUser: CandidateUser,
|
||||
displayLocation: DisplayLocation
|
||||
): FeatureContext = {
|
||||
|
||||
val recipient = getFeatureSwitchRecipient(candidateUser).withCustomFields(
|
||||
"display_location" -> displayLocation.toFsName)
|
||||
new FeatureSwitchResultsFeatureContext(featureSwitches.matchRecipient(recipient))
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
private[configapi] def getFeatureSwitchRecipient(
|
||||
candidateUser: CandidateUser
|
||||
): FeatureSwitchRecipient = {
|
||||
FeatureSwitchRecipient(
|
||||
userId = Some(candidateUser.id),
|
||||
userRoles = None,
|
||||
deviceId = None,
|
||||
guestId = None,
|
||||
languageCode = None,
|
||||
countryCode = None,
|
||||
isVerified = None,
|
||||
clientApplicationId = None,
|
||||
isTwoffice = None
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,35 @@
|
||||
package com.twitter.follow_recommendations.configapi.candidates
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.models.CandidateUser
|
||||
import com.twitter.follow_recommendations.common.models.HasDisplayLocation
|
||||
import com.twitter.follow_recommendations.configapi.params.GlobalParams
|
||||
import com.twitter.servo.util.MemoizingStatsReceiver
|
||||
import com.twitter.timelines.configapi.Config
|
||||
import com.twitter.timelines.configapi.HasParams
|
||||
import com.twitter.timelines.configapi.Params
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
/**
|
||||
* CandidateParamsFactory is primarily used for "producer side" experiments, don't use it on consumer side experiments
|
||||
*/
|
||||
@Singleton
|
||||
class CandidateUserParamsFactory[T <: HasParams with HasDisplayLocation] @Inject() (
|
||||
config: Config,
|
||||
candidateContextFactory: CandidateUserContextFactory,
|
||||
statsReceiver: StatsReceiver) {
|
||||
private val stats = new MemoizingStatsReceiver(statsReceiver.scope("configapi_candidate_params"))
|
||||
def apply(candidateContext: CandidateUser, request: T): CandidateUser = {
|
||||
if (candidateContext.params == Params.Invalid) {
|
||||
if (request.params(GlobalParams.EnableCandidateParamHydrations)) {
|
||||
candidateContext.copy(params =
|
||||
config(candidateContextFactory(candidateContext, request.displayLocation), stats))
|
||||
} else {
|
||||
candidateContext.copy(params = Params.Empty)
|
||||
}
|
||||
} else {
|
||||
candidateContext
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,21 @@
|
||||
package com.twitter.follow_recommendations.configapi.candidates
|
||||
|
||||
import com.google.inject.Inject
|
||||
import com.google.inject.Singleton
|
||||
import com.twitter.follow_recommendations.common.models.CandidateUser
|
||||
import com.twitter.follow_recommendations.common.models.HasDisplayLocation
|
||||
import com.twitter.follow_recommendations.common.base.Transform
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.timelines.configapi.HasParams
|
||||
import com.twitter.util.logging.Logging
|
||||
|
||||
@Singleton
|
||||
class HydrateCandidateParamsTransform[Target <: HasParams with HasDisplayLocation] @Inject() (
|
||||
candidateParamsFactory: CandidateUserParamsFactory[Target])
|
||||
extends Transform[Target, CandidateUser]
|
||||
with Logging {
|
||||
|
||||
def transform(target: Target, candidates: Seq[CandidateUser]): Stitch[Seq[CandidateUser]] = {
|
||||
Stitch.value(candidates.map(candidateParamsFactory.apply(_, target)))
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,8 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"configapi/configapi-core",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,60 @@
|
||||
package com.twitter.follow_recommendations.configapi.common
|
||||
|
||||
import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil.DefinedFeatureName
|
||||
import com.twitter.timelines.configapi.FeatureSwitchOverrideUtil.ValueFeatureName
|
||||
import com.twitter.timelines.configapi.BoundedParam
|
||||
import com.twitter.timelines.configapi.FSBoundedParam
|
||||
import com.twitter.timelines.configapi.FSName
|
||||
import com.twitter.timelines.configapi.HasDurationConversion
|
||||
import com.twitter.timelines.configapi.OptionalOverride
|
||||
import com.twitter.timelines.configapi.Param
|
||||
import com.twitter.util.Duration
|
||||
|
||||
trait FeatureSwitchConfig {
|
||||
def booleanFSParams: Seq[Param[Boolean] with FSName] = Nil
|
||||
|
||||
def intFSParams: Seq[FSBoundedParam[Int]] = Nil
|
||||
|
||||
def longFSParams: Seq[FSBoundedParam[Long]] = Nil
|
||||
|
||||
def doubleFSParams: Seq[FSBoundedParam[Double]] = Nil
|
||||
|
||||
def durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Nil
|
||||
|
||||
def optionalDoubleFSParams: Seq[
|
||||
(BoundedParam[Option[Double]], DefinedFeatureName, ValueFeatureName)
|
||||
] = Nil
|
||||
|
||||
def stringSeqFSParams: Seq[Param[Seq[String]] with FSName] = Nil
|
||||
|
||||
/**
|
||||
* Apply overrides in list when the given FS Key is enabled.
|
||||
* This override type does NOT work with experiments. Params here will be evaluated for every
|
||||
* request IMMEDIATELY, not upon param.apply. If you would like to use an experiment pls use
|
||||
* the primitive type or ENUM overrides.
|
||||
*/
|
||||
def gatedOverridesMap: Map[String, Seq[OptionalOverride[_]]] = Map.empty
|
||||
}
|
||||
|
||||
object FeatureSwitchConfig {
|
||||
def merge(configs: Seq[FeatureSwitchConfig]): FeatureSwitchConfig = new FeatureSwitchConfig {
|
||||
override def booleanFSParams: Seq[Param[Boolean] with FSName] =
|
||||
configs.flatMap(_.booleanFSParams)
|
||||
override def intFSParams: Seq[FSBoundedParam[Int]] =
|
||||
configs.flatMap(_.intFSParams)
|
||||
override def longFSParams: Seq[FSBoundedParam[Long]] =
|
||||
configs.flatMap(_.longFSParams)
|
||||
override def durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] =
|
||||
configs.flatMap(_.durationFSParams)
|
||||
override def gatedOverridesMap: Map[String, Seq[OptionalOverride[_]]] =
|
||||
configs.flatMap(_.gatedOverridesMap).toMap
|
||||
override def doubleFSParams: Seq[FSBoundedParam[Double]] =
|
||||
configs.flatMap(_.doubleFSParams)
|
||||
override def optionalDoubleFSParams: Seq[
|
||||
(BoundedParam[Option[Double]], DefinedFeatureName, ValueFeatureName)
|
||||
] =
|
||||
configs.flatMap(_.optionalDoubleFSParams)
|
||||
override def stringSeqFSParams: Seq[Param[Seq[String]] with FSName] =
|
||||
configs.flatMap(_.stringSeqFSParams)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,10 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"configapi/configapi-core",
|
||||
"configapi/configapi-decider",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,51 @@
|
||||
package com.twitter.follow_recommendations.configapi.deciders
|
||||
|
||||
import com.twitter.servo.decider.DeciderKeyEnum
|
||||
|
||||
object DeciderKey extends DeciderKeyEnum {
|
||||
val EnableDiffyModuleDarkReading = Value("enable_diffy_module_dark_reading")
|
||||
val EnableRecommendations = Value("enable_recommendations")
|
||||
val EnableScoreUserCandidates = Value("enable_score_user_candidates")
|
||||
val EnableProfileSidebarProduct = Value("enable_profile_sidebar_product")
|
||||
val EnableMagicRecsProduct = Value("enable_magic_recs_product")
|
||||
val EnableRuxLandingPageProduct = Value("enable_rux_landing_page_product")
|
||||
val EnableRuxPymkProduct = Value("enable_rux_pymk_product")
|
||||
val EnableProfileBonusFollowProduct = Value("enable_profile_bonus_follow_product")
|
||||
val EnableElectionExploreWtfProduct = Value("enable_election_explore_wtf_product")
|
||||
val EnableClusterFollowProduct = Value("enable_cluster_follow_product")
|
||||
val EnableHomeTimelineProduct = Value("enable_home_timeline_product")
|
||||
val EnableHtlBonusFollowProduct = Value("enable_htl_bonus_follow_product")
|
||||
val EnableExploreTabProduct = Value("enable_explore_tab_product")
|
||||
val EnableSidebarProduct = Value("enable_sidebar_product")
|
||||
val EnableNuxPymkProduct = Value("enable_nux_pymk_product")
|
||||
val EnableNuxInterestsProduct = Value("enable_nux_interests_product")
|
||||
val EnableNuxTopicBonusFollowProduct = Value("enable_nux_topic_bonus_follow_product")
|
||||
val EnableCampaignFormProduct = Value("enable_campaign_form_product")
|
||||
val EnableReactiveFollowProduct = Value("enable_reactive_follow_product")
|
||||
val EnableIndiaCovid19CuratedAccountsWtfProduct = Value(
|
||||
"enable_india_covid19_curated_accounts_wtf_product")
|
||||
val EnableAbUploadProduct = Value("enable_ab_upload_product")
|
||||
val EnablePeolePlusPlusProduct = Value("enable_people_plus_plus_product")
|
||||
val EnableTweetNotificationRecsProduct = Value("enable_tweet_notification_recs_product")
|
||||
val EnableProfileDeviceFollow = Value("enable_profile_device_follow_product")
|
||||
val EnableRecosBackfillProduct = Value("enable_recos_backfill_product")
|
||||
val EnablePostNuxFollowTaskProduct = Value("enable_post_nux_follow_task_product")
|
||||
val EnableCuratedSpaceHostsProduct = Value("enable_curated_space_hosts_product")
|
||||
val EnableNuxGeoCategoryProduct = Value("enable_nux_geo_category_product")
|
||||
val EnableNuxInterestsCategoryProduct = Value("enable_nux_interests_category_product")
|
||||
val EnableNuxPymkCategoryProduct = Value("enable_nux_pymk_category_product")
|
||||
val EnableHomeTimelineTweetRecsProduct = Value("enable_home_timeline_tweet_recs_product")
|
||||
val EnableHtlBulkFriendFollowsProduct = Value("enable_htl_bulk_friend_follows_product")
|
||||
val EnableNuxAutoFollowProduct = Value("enable_nux_auto_follow_product")
|
||||
val EnableSearchBonusFollowProduct = Value("enable_search_bonus_follow_product")
|
||||
val EnableFetchUserInRequestBuilder = Value("enable_fetch_user_in_request_builder")
|
||||
val EnableProductMixerMagicRecsProduct = Value("enable_product_mixer_magic_recs_product")
|
||||
val EnableHomeTimelineReverseChronProduct = Value("enable_home_timeline_reverse_chron_product")
|
||||
val EnableProductMixerPipelineMagicRecsDarkRead = Value(
|
||||
"enable_product_mixer_pipeline_magic_recs_dark_read")
|
||||
val EnableExperimentalCaching = Value("enable_experimental_caching")
|
||||
val EnableDistributedCaching = Value("enable_distributed_caching")
|
||||
val EnableGizmoduckCaching = Value("enable_gizmoduck_caching")
|
||||
val EnableTrafficDarkReading = Value("enable_traffic_dark_reading")
|
||||
val EnableGraphFeatureServiceRequests = Value("enable_graph_feature_service_requests")
|
||||
}
|
||||
@ -0,0 +1,8 @@
|
||||
package com.twitter.follow_recommendations.configapi.deciders
|
||||
|
||||
import com.twitter.timelines.configapi.Param
|
||||
|
||||
object DeciderParams {
|
||||
object EnableRecommendations extends Param[Boolean](false)
|
||||
object EnableScoreUserCandidates extends Param[Boolean](false)
|
||||
}
|
||||
@ -0,0 +1,13 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"configapi/configapi-core",
|
||||
"configapi/configapi-decider",
|
||||
"configapi/configapi-featureswitches:v2",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/deciders",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,35 @@
|
||||
package com.twitter.follow_recommendations.configapi.params
|
||||
|
||||
import com.twitter.follow_recommendations.models.CandidateSourceType
|
||||
import com.twitter.timelines.configapi.FSEnumParam
|
||||
import com.twitter.timelines.configapi.FSParam
|
||||
|
||||
/**
|
||||
* When adding Producer side experiments, make sure to register the FS Key in [[ProducerFeatureFilter]]
|
||||
* in [[FeatureSwitchesModule]], otherwise, the FS will not work.
|
||||
*/
|
||||
object GlobalParams {
|
||||
|
||||
object EnableCandidateParamHydrations
|
||||
extends FSParam[Boolean]("frs_receiver_enable_candidate_params", false)
|
||||
|
||||
object KeepUserCandidate
|
||||
extends FSParam[Boolean]("frs_receiver_holdback_keep_user_candidate", true)
|
||||
|
||||
object KeepSocialUserCandidate
|
||||
extends FSParam[Boolean]("frs_receiver_holdback_keep_social_user_candidate", true)
|
||||
|
||||
case object EnableGFSSocialProofTransform
|
||||
extends FSParam("social_proof_transform_use_graph_feature_service", true)
|
||||
|
||||
case object EnableWhoToFollowProducts extends FSParam("who_to_follow_product_enabled", true)
|
||||
|
||||
case object CandidateSourcesToFilter
|
||||
extends FSEnumParam[CandidateSourceType.type](
|
||||
"candidate_sources_type_filter_id",
|
||||
CandidateSourceType.None,
|
||||
CandidateSourceType)
|
||||
|
||||
object EnableRecommendationFlowLogs
|
||||
extends FSParam[Boolean]("frs_recommendation_flow_logs_enabled", false)
|
||||
}
|
||||
@ -0,0 +1,29 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/com/google/inject/extensions:guice-assistedinject",
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"3rdparty/jvm/net/codingwell:scala-guice",
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"decider/src/main/scala",
|
||||
"finagle/finagle-core/src/main",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"finatra/thrift/src/main/scala/com/twitter/finatra/thrift",
|
||||
"finatra/thrift/src/main/scala/com/twitter/finatra/thrift:controller",
|
||||
"finatra/thrift/src/main/scala/com/twitter/finatra/thrift/exceptions",
|
||||
"finatra/thrift/src/main/scala/com/twitter/finatra/thrift/filters",
|
||||
"finatra/thrift/src/main/scala/com/twitter/finatra/thrift/modules",
|
||||
"finatra/thrift/src/main/scala/com/twitter/finatra/thrift/response",
|
||||
"finatra/thrift/src/main/scala/com/twitter/finatra/thrift/routing",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/services",
|
||||
"follow-recommendations-service/thrift/src/main/thrift:thrift-scala",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/service/debug_query",
|
||||
"scrooge/scrooge-core/src/main/scala",
|
||||
"util/util-core:scala",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,25 @@
|
||||
package com.twitter.follow_recommendations.controllers
|
||||
|
||||
import com.twitter.follow_recommendations.common.models._
|
||||
import com.twitter.follow_recommendations.configapi.ParamsFactory
|
||||
import com.twitter.follow_recommendations.models.CandidateUserDebugParams
|
||||
import com.twitter.follow_recommendations.models.FeatureValue
|
||||
import com.twitter.follow_recommendations.{thriftscala => t}
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class CandidateUserDebugParamsBuilder @Inject() (paramsFactory: ParamsFactory) {
|
||||
def fromThrift(req: t.ScoringUserRequest): CandidateUserDebugParams = {
|
||||
val clientContext = ClientContextConverter.fromThrift(req.clientContext)
|
||||
val displayLocation = DisplayLocation.fromThrift(req.displayLocation)
|
||||
|
||||
CandidateUserDebugParams(req.candidates.map { candidate =>
|
||||
candidate.userId -> paramsFactory(
|
||||
clientContext,
|
||||
displayLocation,
|
||||
candidate.featureOverrides
|
||||
.map(_.mapValues(FeatureValue.fromThrift).toMap).getOrElse(Map.empty))
|
||||
}.toMap)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
package com.twitter.follow_recommendations.controllers
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.models.ClientContextConverter
|
||||
import com.twitter.follow_recommendations.common.models.DisplayLocation
|
||||
import com.twitter.follow_recommendations.models.DebugParams
|
||||
import com.twitter.follow_recommendations.models.DisplayContext
|
||||
import com.twitter.follow_recommendations.models.RecommendationRequest
|
||||
import com.twitter.follow_recommendations.{thriftscala => t}
|
||||
import com.twitter.gizmoduck.thriftscala.UserType
|
||||
import com.twitter.stitch.Stitch
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class RecommendationRequestBuilder @Inject() (
|
||||
requestBuilderUserFetcher: RequestBuilderUserFetcher,
|
||||
statsReceiver: StatsReceiver) {
|
||||
private val scopedStats = statsReceiver.scope(this.getClass.getSimpleName)
|
||||
private val isSoftUserCounter = scopedStats.counter("is_soft_user")
|
||||
|
||||
def fromThrift(tRequest: t.RecommendationRequest): Stitch[RecommendationRequest] = {
|
||||
requestBuilderUserFetcher.fetchUser(tRequest.clientContext.userId).map { userOpt =>
|
||||
val isSoftUser = userOpt.exists(_.userType == UserType.Soft)
|
||||
if (isSoftUser) isSoftUserCounter.incr()
|
||||
RecommendationRequest(
|
||||
clientContext = ClientContextConverter.fromThrift(tRequest.clientContext),
|
||||
displayLocation = DisplayLocation.fromThrift(tRequest.displayLocation),
|
||||
displayContext = tRequest.displayContext.map(DisplayContext.fromThrift),
|
||||
maxResults = tRequest.maxResults,
|
||||
cursor = tRequest.cursor,
|
||||
excludedIds = tRequest.excludedIds,
|
||||
fetchPromotedContent = tRequest.fetchPromotedContent,
|
||||
debugParams = tRequest.debugParams.map(DebugParams.fromThrift),
|
||||
userLocationState = tRequest.userLocationState,
|
||||
isSoftUser = isSoftUser
|
||||
)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,48 @@
|
||||
package com.twitter.follow_recommendations.controllers
|
||||
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.decider.SimpleRecipient
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.base.StatsUtil
|
||||
import com.twitter.follow_recommendations.configapi.deciders.DeciderKey
|
||||
import com.twitter.gizmoduck.thriftscala.LookupContext
|
||||
import com.twitter.gizmoduck.thriftscala.User
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.stitch.gizmoduck.Gizmoduck
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class RequestBuilderUserFetcher @Inject() (
|
||||
gizmoduck: Gizmoduck,
|
||||
statsReceiver: StatsReceiver,
|
||||
decider: Decider) {
|
||||
private val scopedStats = statsReceiver.scope(this.getClass.getSimpleName)
|
||||
|
||||
def fetchUser(userIdOpt: Option[Long]): Stitch[Option[User]] = {
|
||||
userIdOpt match {
|
||||
case Some(userId) if enableDecider(userId) =>
|
||||
val stitch = gizmoduck
|
||||
.getUserById(
|
||||
userId = userId,
|
||||
context = LookupContext(
|
||||
forUserId = Some(userId),
|
||||
includeProtected = true,
|
||||
includeSoftUsers = true
|
||||
)
|
||||
).map(user => Some(user))
|
||||
StatsUtil
|
||||
.profileStitch(stitch, scopedStats)
|
||||
.handle {
|
||||
case _: Throwable => None
|
||||
}
|
||||
case _ => Stitch.None
|
||||
}
|
||||
}
|
||||
|
||||
private def enableDecider(userId: Long): Boolean = {
|
||||
decider.isAvailable(
|
||||
DeciderKey.EnableFetchUserInRequestBuilder.toString,
|
||||
Some(SimpleRecipient(userId)))
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,53 @@
|
||||
package com.twitter.follow_recommendations.controllers
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.models.CandidateUser
|
||||
import com.twitter.follow_recommendations.common.models.ClientContextConverter
|
||||
import com.twitter.follow_recommendations.common.models.DebugOptions
|
||||
import com.twitter.follow_recommendations.common.models.DisplayLocation
|
||||
import com.twitter.follow_recommendations.models.DebugParams
|
||||
import com.twitter.follow_recommendations.models.ScoringUserRequest
|
||||
import com.twitter.timelines.configapi.Params
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
import com.twitter.follow_recommendations.{thriftscala => t}
|
||||
import com.twitter.gizmoduck.thriftscala.UserType
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
@Singleton
|
||||
class ScoringUserRequestBuilder @Inject() (
|
||||
requestBuilderUserFetcher: RequestBuilderUserFetcher,
|
||||
candidateUserDebugParamsBuilder: CandidateUserDebugParamsBuilder,
|
||||
statsReceiver: StatsReceiver) {
|
||||
private val scopedStats = statsReceiver.scope(this.getClass.getSimpleName)
|
||||
private val isSoftUserCounter = scopedStats.counter("is_soft_user")
|
||||
|
||||
def fromThrift(req: t.ScoringUserRequest): Stitch[ScoringUserRequest] = {
|
||||
requestBuilderUserFetcher.fetchUser(req.clientContext.userId).map { userOpt =>
|
||||
val isSoftUser = userOpt.exists(_.userType == UserType.Soft)
|
||||
if (isSoftUser) isSoftUserCounter.incr()
|
||||
|
||||
val candidateUsersParamsMap = candidateUserDebugParamsBuilder.fromThrift(req)
|
||||
val candidates = req.candidates.map { candidate =>
|
||||
CandidateUser
|
||||
.fromUserRecommendation(candidate).copy(params =
|
||||
candidateUsersParamsMap.paramsMap.getOrElse(candidate.userId, Params.Invalid))
|
||||
}
|
||||
|
||||
ScoringUserRequest(
|
||||
clientContext = ClientContextConverter.fromThrift(req.clientContext),
|
||||
displayLocation = DisplayLocation.fromThrift(req.displayLocation),
|
||||
params = Params.Empty,
|
||||
debugOptions = req.debugParams.map(DebugOptions.fromDebugParamsThrift),
|
||||
recentFollowedUserIds = None,
|
||||
recentFollowedByUserIds = None,
|
||||
wtfImpressions = None,
|
||||
similarToUserIds = Nil,
|
||||
candidates = candidates,
|
||||
debugParams = req.debugParams.map(DebugParams.fromThrift),
|
||||
isSoftUser = isSoftUser
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
package com.twitter.follow_recommendations.controllers
|
||||
|
||||
import com.twitter.finatra.thrift.Controller
|
||||
import com.twitter.follow_recommendations.configapi.ParamsFactory
|
||||
import com.twitter.follow_recommendations.services.ProductPipelineSelector
|
||||
import com.twitter.follow_recommendations.services.UserScoringService
|
||||
import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService
|
||||
import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService._
|
||||
import com.twitter.stitch.Stitch
|
||||
import javax.inject.Inject
|
||||
|
||||
class ThriftController @Inject() (
|
||||
userScoringService: UserScoringService,
|
||||
recommendationRequestBuilder: RecommendationRequestBuilder,
|
||||
scoringUserRequestBuilder: ScoringUserRequestBuilder,
|
||||
productPipelineSelector: ProductPipelineSelector,
|
||||
paramsFactory: ParamsFactory)
|
||||
extends Controller(FollowRecommendationsThriftService) {
|
||||
|
||||
handle(GetRecommendations) { args: GetRecommendations.Args =>
|
||||
val stitch = recommendationRequestBuilder.fromThrift(args.request).flatMap { request =>
|
||||
val params = paramsFactory(
|
||||
request.clientContext,
|
||||
request.displayLocation,
|
||||
request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty))
|
||||
productPipelineSelector.selectPipeline(request, params).map(_.toThrift)
|
||||
}
|
||||
Stitch.run(stitch)
|
||||
}
|
||||
|
||||
handle(ScoreUserCandidates) { args: ScoreUserCandidates.Args =>
|
||||
val stitch = scoringUserRequestBuilder.fromThrift(args.request).flatMap { request =>
|
||||
val params = paramsFactory(
|
||||
request.clientContext,
|
||||
request.displayLocation,
|
||||
request.debugParams.flatMap(_.featureOverrides).getOrElse(Map.empty))
|
||||
userScoringService.get(request.copy(params = params)).map(_.toThrift)
|
||||
}
|
||||
Stitch.run(stitch)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,19 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/com/google/inject/extensions:guice-assistedinject",
|
||||
"3rdparty/jvm/net/codingwell:scala-guice",
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/promoted_accounts",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,112 @@
|
||||
package com.twitter.follow_recommendations.flows.ads
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.base.EnrichedCandidateSource
|
||||
import com.twitter.follow_recommendations.common.base.IdentityRanker
|
||||
import com.twitter.follow_recommendations.common.base.IdentityTransform
|
||||
import com.twitter.follow_recommendations.common.base.ParamPredicate
|
||||
import com.twitter.follow_recommendations.common.base.Predicate
|
||||
import com.twitter.follow_recommendations.common.base.Ranker
|
||||
import com.twitter.follow_recommendations.common.base.RecommendationFlow
|
||||
import com.twitter.follow_recommendations.common.base.RecommendationResultsConfig
|
||||
import com.twitter.follow_recommendations.common.base.Transform
|
||||
import com.twitter.follow_recommendations.common.base.TruePredicate
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.promoted_accounts.PromotedAccountsCandidateSource
|
||||
import com.twitter.follow_recommendations.common.models.CandidateUser
|
||||
import com.twitter.follow_recommendations.common.predicates.ExcludedUserIdPredicate
|
||||
import com.twitter.follow_recommendations.common.transforms.tracking_token.TrackingTokenTransform
|
||||
import com.twitter.inject.annotations.Flag
|
||||
import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource
|
||||
import com.twitter.util.Duration
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class PromotedAccountsFlow @Inject() (
|
||||
promotedAccountsCandidateSource: PromotedAccountsCandidateSource,
|
||||
trackingTokenTransform: TrackingTokenTransform,
|
||||
baseStatsReceiver: StatsReceiver,
|
||||
@Flag("fetch_prod_promoted_accounts") fetchProductionPromotedAccounts: Boolean)
|
||||
extends RecommendationFlow[PromotedAccountsFlowRequest, CandidateUser] {
|
||||
|
||||
protected override def targetEligibility: Predicate[PromotedAccountsFlowRequest] =
|
||||
new ParamPredicate[PromotedAccountsFlowRequest](
|
||||
PromotedAccountsFlowParams.TargetEligibility
|
||||
)
|
||||
|
||||
protected override def candidateSources(
|
||||
target: PromotedAccountsFlowRequest
|
||||
): Seq[CandidateSource[PromotedAccountsFlowRequest, CandidateUser]] = {
|
||||
import EnrichedCandidateSource._
|
||||
val candidateSourceStats = statsReceiver.scope("candidate_sources")
|
||||
val budget: Duration = target.params(PromotedAccountsFlowParams.FetchCandidateSourceBudget)
|
||||
val candidateSources = Seq(
|
||||
promotedAccountsCandidateSource
|
||||
.mapKeys[PromotedAccountsFlowRequest](r =>
|
||||
Seq(r.toAdsRequest(fetchProductionPromotedAccounts)))
|
||||
.mapValue(PromotedAccountsUtil.toCandidateUser)
|
||||
).map { candidateSource =>
|
||||
candidateSource
|
||||
.failOpenWithin(budget, candidateSourceStats).observe(candidateSourceStats)
|
||||
}
|
||||
candidateSources
|
||||
}
|
||||
|
||||
protected override def preRankerCandidateFilter: Predicate[
|
||||
(PromotedAccountsFlowRequest, CandidateUser)
|
||||
] = {
|
||||
val preRankerFilterStats = statsReceiver.scope("pre_ranker")
|
||||
ExcludedUserIdPredicate.observe(preRankerFilterStats.scope("exclude_user_id_predicate"))
|
||||
}
|
||||
|
||||
/**
|
||||
* rank the candidates
|
||||
*/
|
||||
protected override def selectRanker(
|
||||
target: PromotedAccountsFlowRequest
|
||||
): Ranker[PromotedAccountsFlowRequest, CandidateUser] = {
|
||||
new IdentityRanker[PromotedAccountsFlowRequest, CandidateUser]
|
||||
}
|
||||
|
||||
/**
|
||||
* transform the candidates after ranking (e.g. dedupping, grouping and etc)
|
||||
*/
|
||||
protected override def postRankerTransform: Transform[
|
||||
PromotedAccountsFlowRequest,
|
||||
CandidateUser
|
||||
] = {
|
||||
new IdentityTransform[PromotedAccountsFlowRequest, CandidateUser]
|
||||
}
|
||||
|
||||
/**
|
||||
* filter invalid candidates before returning the results.
|
||||
*
|
||||
* Some heavy filters e.g. SGS filter could be applied in this step
|
||||
*/
|
||||
protected override def validateCandidates: Predicate[
|
||||
(PromotedAccountsFlowRequest, CandidateUser)
|
||||
] = {
|
||||
new TruePredicate[(PromotedAccountsFlowRequest, CandidateUser)]
|
||||
}
|
||||
|
||||
/**
|
||||
* transform the candidates into results and return
|
||||
*/
|
||||
protected override def transformResults: Transform[PromotedAccountsFlowRequest, CandidateUser] = {
|
||||
trackingTokenTransform
|
||||
}
|
||||
|
||||
/**
|
||||
* configuration for recommendation results
|
||||
*/
|
||||
protected override def resultsConfig(
|
||||
target: PromotedAccountsFlowRequest
|
||||
): RecommendationResultsConfig = {
|
||||
RecommendationResultsConfig(
|
||||
target.params(PromotedAccountsFlowParams.ResultSizeParam),
|
||||
target.params(PromotedAccountsFlowParams.BatchSizeParam)
|
||||
)
|
||||
}
|
||||
|
||||
override val statsReceiver: StatsReceiver = baseStatsReceiver.scope("promoted_accounts_flow")
|
||||
}
|
||||
@ -0,0 +1,19 @@
|
||||
package com.twitter.follow_recommendations.flows.ads
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.timelines.configapi.Param
|
||||
import com.twitter.util.Duration
|
||||
|
||||
abstract class PromotedAccountsFlowParams[A](default: A) extends Param[A](default) {
|
||||
override val statName: String = "ads/" + this.getClass.getSimpleName
|
||||
}
|
||||
|
||||
object PromotedAccountsFlowParams {
|
||||
|
||||
// number of total slots returned to the end user, available to put ads
|
||||
case object TargetEligibility extends PromotedAccountsFlowParams[Boolean](true)
|
||||
case object ResultSizeParam extends PromotedAccountsFlowParams[Int](Int.MaxValue)
|
||||
case object BatchSizeParam extends PromotedAccountsFlowParams[Int](Int.MaxValue)
|
||||
case object FetchCandidateSourceBudget
|
||||
extends PromotedAccountsFlowParams[Duration](1000.millisecond)
|
||||
|
||||
}
|
||||
@ -0,0 +1,33 @@
|
||||
package com.twitter.follow_recommendations.flows.ads
|
||||
import com.twitter.follow_recommendations.common.clients.adserver.AdRequest
|
||||
import com.twitter.follow_recommendations.common.models.DisplayLocation
|
||||
import com.twitter.follow_recommendations.common.models.HasDisplayLocation
|
||||
import com.twitter.follow_recommendations.common.models.HasExcludedUserIds
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.ClientContext
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext
|
||||
import com.twitter.timelines.configapi.HasParams
|
||||
import com.twitter.timelines.configapi.Params
|
||||
|
||||
case class PromotedAccountsFlowRequest(
|
||||
override val clientContext: ClientContext,
|
||||
override val params: Params,
|
||||
displayLocation: DisplayLocation,
|
||||
profileId: Option[Long],
|
||||
// note we also add userId and profileId to excludeUserIds
|
||||
excludeIds: Seq[Long])
|
||||
extends HasParams
|
||||
with HasClientContext
|
||||
with HasExcludedUserIds
|
||||
with HasDisplayLocation {
|
||||
def toAdsRequest(fetchProductionPromotedAccounts: Boolean): AdRequest = {
|
||||
AdRequest(
|
||||
clientContext = clientContext,
|
||||
displayLocation = displayLocation,
|
||||
isTest = Some(!fetchProductionPromotedAccounts),
|
||||
profileUserId = profileId
|
||||
)
|
||||
}
|
||||
override val excludedUserIds: Seq[Long] = {
|
||||
excludeIds ++ clientContext.userId.toSeq ++ profileId.toSeq
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,28 @@
|
||||
package com.twitter.follow_recommendations.flows.ads
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.promoted_accounts.PromotedCandidateUser
|
||||
import com.twitter.follow_recommendations.common.models.AccountProof
|
||||
import com.twitter.follow_recommendations.common.models.AdMetadata
|
||||
import com.twitter.follow_recommendations.common.models.CandidateUser
|
||||
import com.twitter.follow_recommendations.common.models.Reason
|
||||
import com.twitter.follow_recommendations.common.models.UserCandidateSourceDetails
|
||||
|
||||
object PromotedAccountsUtil {
|
||||
def toCandidateUser(promotedCandidateUser: PromotedCandidateUser): CandidateUser = {
|
||||
CandidateUser(
|
||||
id = promotedCandidateUser.id,
|
||||
score = None,
|
||||
adMetadata =
|
||||
Some(AdMetadata(promotedCandidateUser.position, promotedCandidateUser.adImpression)),
|
||||
reason = Some(
|
||||
Reason(
|
||||
accountProof = Some(AccountProof(followProof = Some(promotedCandidateUser.followProof))))
|
||||
),
|
||||
userCandidateSourceDetails = Some(
|
||||
UserCandidateSourceDetails(
|
||||
promotedCandidateUser.primaryCandidateSource,
|
||||
Map.empty,
|
||||
Map.empty,
|
||||
None))
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,32 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/com/google/inject/extensions:guice-assistedinject",
|
||||
"3rdparty/jvm/net/codingwell:scala-guice",
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,202 @@
|
||||
package com.twitter.follow_recommendations.flows.content_recommender_flow
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.base.EnrichedCandidateSource
|
||||
import com.twitter.follow_recommendations.common.base.GatedPredicateBase
|
||||
import com.twitter.follow_recommendations.common.base.ParamPredicate
|
||||
import com.twitter.follow_recommendations.common.base.Predicate
|
||||
import com.twitter.follow_recommendations.common.base.Ranker
|
||||
import com.twitter.follow_recommendations.common.base.RecommendationFlow
|
||||
import com.twitter.follow_recommendations.common.base.RecommendationResultsConfig
|
||||
import com.twitter.follow_recommendations.common.base.Transform
|
||||
import com.twitter.follow_recommendations.common.models.CandidateUser
|
||||
import com.twitter.follow_recommendations.common.predicates.ExcludedUserIdPredicate
|
||||
import com.twitter.follow_recommendations.common.predicates.InactivePredicate
|
||||
import com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicate
|
||||
import com.twitter.follow_recommendations.common.predicates.sgs.InvalidRelationshipPredicate
|
||||
import com.twitter.follow_recommendations.common.predicates.sgs.InvalidTargetCandidateRelationshipTypesPredicate
|
||||
import com.twitter.follow_recommendations.common.predicates.sgs.RecentFollowingPredicate
|
||||
import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.WeightedCandidateSourceRanker
|
||||
import com.twitter.follow_recommendations.common.transforms.dedup.DedupTransform
|
||||
import com.twitter.follow_recommendations.common.transforms.tracking_token.TrackingTokenTransform
|
||||
import com.twitter.follow_recommendations.utils.CandidateSourceHoldbackUtil
|
||||
import com.twitter.follow_recommendations.utils.RecommendationFlowBaseSideEffectsUtil
|
||||
import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource
|
||||
import com.twitter.product_mixer.core.quality_factor.BoundsWithDefault
|
||||
import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactor
|
||||
import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactorConfig
|
||||
import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactorObserver
|
||||
import com.twitter.product_mixer.core.quality_factor.QualityFactorObserver
|
||||
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class ContentRecommenderFlow @Inject() (
|
||||
contentRecommenderFlowCandidateSourceRegistry: ContentRecommenderFlowCandidateSourceRegistry,
|
||||
recentFollowingPredicate: RecentFollowingPredicate,
|
||||
gizmoduckPredicate: GizmoduckPredicate,
|
||||
inactivePredicate: InactivePredicate,
|
||||
sgsPredicate: InvalidTargetCandidateRelationshipTypesPredicate,
|
||||
invalidRelationshipPredicate: InvalidRelationshipPredicate,
|
||||
trackingTokenTransform: TrackingTokenTransform,
|
||||
baseStatsReceiver: StatsReceiver)
|
||||
extends RecommendationFlow[ContentRecommenderRequest, CandidateUser]
|
||||
with RecommendationFlowBaseSideEffectsUtil[ContentRecommenderRequest, CandidateUser]
|
||||
with CandidateSourceHoldbackUtil {
|
||||
|
||||
override val statsReceiver: StatsReceiver = baseStatsReceiver.scope("content_recommender_flow")
|
||||
|
||||
override val qualityFactorObserver: Option[QualityFactorObserver] = {
|
||||
val config = LinearLatencyQualityFactorConfig(
|
||||
qualityFactorBounds =
|
||||
BoundsWithDefault(minInclusive = 0.1, maxInclusive = 1.0, default = 1.0),
|
||||
initialDelay = 60.seconds,
|
||||
targetLatency = 100.milliseconds,
|
||||
targetLatencyPercentile = 95.0,
|
||||
delta = 0.001
|
||||
)
|
||||
val qualityFactor = LinearLatencyQualityFactor(config)
|
||||
val observer = LinearLatencyQualityFactorObserver(qualityFactor)
|
||||
statsReceiver.provideGauge("quality_factor")(qualityFactor.currentValue.toFloat)
|
||||
Some(observer)
|
||||
}
|
||||
|
||||
protected override def targetEligibility: Predicate[ContentRecommenderRequest] =
|
||||
new ParamPredicate[ContentRecommenderRequest](
|
||||
ContentRecommenderParams.TargetEligibility
|
||||
)
|
||||
|
||||
protected override def candidateSources(
|
||||
target: ContentRecommenderRequest
|
||||
): Seq[CandidateSource[ContentRecommenderRequest, CandidateUser]] = {
|
||||
import EnrichedCandidateSource._
|
||||
val identifiers = ContentRecommenderFlowCandidateSourceWeights.getWeights(target.params).keySet
|
||||
val selected = contentRecommenderFlowCandidateSourceRegistry.select(identifiers)
|
||||
val budget =
|
||||
target.params(ContentRecommenderParams.FetchCandidateSourceBudgetInMillisecond).millisecond
|
||||
filterCandidateSources(target, selected.map(c => c.failOpenWithin(budget, statsReceiver)).toSeq)
|
||||
}
|
||||
|
||||
protected override val preRankerCandidateFilter: Predicate[
|
||||
(ContentRecommenderRequest, CandidateUser)
|
||||
] = {
|
||||
val preRankerFilterStats = statsReceiver.scope("pre_ranker")
|
||||
val recentFollowingPredicateStats = preRankerFilterStats.scope("recent_following_predicate")
|
||||
val invalidRelationshipPredicateStats =
|
||||
preRankerFilterStats.scope("invalid_relationship_predicate")
|
||||
|
||||
object recentFollowingGatedPredicate
|
||||
extends GatedPredicateBase[(ContentRecommenderRequest, CandidateUser)](
|
||||
recentFollowingPredicate,
|
||||
recentFollowingPredicateStats
|
||||
) {
|
||||
override def gate(item: (ContentRecommenderRequest, CandidateUser)): Boolean =
|
||||
item._1.params(ContentRecommenderParams.EnableRecentFollowingPredicate)
|
||||
}
|
||||
|
||||
object invalidRelationshipGatedPredicate
|
||||
extends GatedPredicateBase[(ContentRecommenderRequest, CandidateUser)](
|
||||
invalidRelationshipPredicate,
|
||||
invalidRelationshipPredicateStats
|
||||
) {
|
||||
override def gate(item: (ContentRecommenderRequest, CandidateUser)): Boolean =
|
||||
item._1.params(ContentRecommenderParams.EnableInvalidRelationshipPredicate)
|
||||
}
|
||||
|
||||
ExcludedUserIdPredicate
|
||||
.observe(preRankerFilterStats.scope("exclude_user_id_predicate"))
|
||||
.andThen(recentFollowingGatedPredicate.observe(recentFollowingPredicateStats))
|
||||
.andThen(invalidRelationshipGatedPredicate.observe(invalidRelationshipPredicateStats))
|
||||
}
|
||||
|
||||
/**
|
||||
* rank the candidates
|
||||
*/
|
||||
protected override def selectRanker(
|
||||
target: ContentRecommenderRequest
|
||||
): Ranker[ContentRecommenderRequest, CandidateUser] = {
|
||||
val rankersStatsReceiver = statsReceiver.scope("rankers")
|
||||
WeightedCandidateSourceRanker
|
||||
.build[ContentRecommenderRequest](
|
||||
ContentRecommenderFlowCandidateSourceWeights.getWeights(target.params),
|
||||
randomSeed = target.getRandomizationSeed
|
||||
).observe(rankersStatsReceiver.scope("weighted_candidate_source_ranker"))
|
||||
}
|
||||
|
||||
/**
|
||||
* transform the candidates after ranking
|
||||
*/
|
||||
protected override def postRankerTransform: Transform[
|
||||
ContentRecommenderRequest,
|
||||
CandidateUser
|
||||
] = {
|
||||
new DedupTransform[ContentRecommenderRequest, CandidateUser]
|
||||
.observe(statsReceiver.scope("dedupping"))
|
||||
}
|
||||
|
||||
protected override def validateCandidates: Predicate[
|
||||
(ContentRecommenderRequest, CandidateUser)
|
||||
] = {
|
||||
val stats = statsReceiver.scope("validate_candidates")
|
||||
val gizmoduckPredicateStats = stats.scope("gizmoduck_predicate")
|
||||
val inactivePredicateStats = stats.scope("inactive_predicate")
|
||||
val sgsPredicateStats = stats.scope("sgs_predicate")
|
||||
|
||||
val includeGizmoduckPredicate =
|
||||
new ParamPredicate[ContentRecommenderRequest](
|
||||
ContentRecommenderParams.EnableGizmoduckPredicate)
|
||||
.map[(ContentRecommenderRequest, CandidateUser)] {
|
||||
case (request: ContentRecommenderRequest, _) =>
|
||||
request
|
||||
}
|
||||
|
||||
val includeInactivePredicate =
|
||||
new ParamPredicate[ContentRecommenderRequest](
|
||||
ContentRecommenderParams.EnableInactivePredicate)
|
||||
.map[(ContentRecommenderRequest, CandidateUser)] {
|
||||
case (request: ContentRecommenderRequest, _) =>
|
||||
request
|
||||
}
|
||||
|
||||
val includeInvalidTargetCandidateRelationshipTypesPredicate =
|
||||
new ParamPredicate[ContentRecommenderRequest](
|
||||
ContentRecommenderParams.EnableInvalidTargetCandidateRelationshipPredicate)
|
||||
.map[(ContentRecommenderRequest, CandidateUser)] {
|
||||
case (request: ContentRecommenderRequest, _) =>
|
||||
request
|
||||
}
|
||||
|
||||
Predicate
|
||||
.andConcurrently[(ContentRecommenderRequest, CandidateUser)](
|
||||
Seq(
|
||||
gizmoduckPredicate.observe(gizmoduckPredicateStats).gate(includeGizmoduckPredicate),
|
||||
inactivePredicate.observe(inactivePredicateStats).gate(includeInactivePredicate),
|
||||
sgsPredicate
|
||||
.observe(sgsPredicateStats).gate(
|
||||
includeInvalidTargetCandidateRelationshipTypesPredicate),
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* transform the candidates into results and return
|
||||
*/
|
||||
protected override def transformResults: Transform[ContentRecommenderRequest, CandidateUser] = {
|
||||
trackingTokenTransform
|
||||
}
|
||||
|
||||
/**
|
||||
* configuration for recommendation results
|
||||
*/
|
||||
protected override def resultsConfig(
|
||||
target: ContentRecommenderRequest
|
||||
): RecommendationResultsConfig = {
|
||||
RecommendationResultsConfig(
|
||||
target.maxResults.getOrElse(target.params(ContentRecommenderParams.ResultSizeParam)),
|
||||
target.params(ContentRecommenderParams.BatchSizeParam)
|
||||
)
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,78 @@
|
||||
package com.twitter.follow_recommendations.flows.content_recommender_flow
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.base.CandidateSourceRegistry
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardEmailBookSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardPhoneBookSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReverseEmailBookSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReversePhoneBookSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountryBackFillSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountrySource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.real_graph.RealGraphOonV2Source
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RepeatedProfileVisitsSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentFollowingSimilarUsersSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.socialgraph.RecentFollowingRecentFollowingExpansionSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStrongTiePredictionSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.user_user_graph.UserUserGraphCandidateSource
|
||||
import com.twitter.follow_recommendations.common.models.CandidateUser
|
||||
import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource
|
||||
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class ContentRecommenderFlowCandidateSourceRegistry @Inject() (
|
||||
// social based
|
||||
forwardPhoneBookSource: ForwardPhoneBookSource,
|
||||
forwardEmailBookSource: ForwardEmailBookSource,
|
||||
reversePhoneBookSource: ReversePhoneBookSource,
|
||||
reverseEmailBookSource: ReverseEmailBookSource,
|
||||
offlineStrongTiePredictionSource: OfflineStrongTiePredictionSource,
|
||||
triangularLoopsSource: TriangularLoopsSource,
|
||||
userUserGraphCandidateSource: UserUserGraphCandidateSource,
|
||||
realGraphOonSource: RealGraphOonV2Source,
|
||||
recentFollowingRecentFollowingExpansionSource: RecentFollowingRecentFollowingExpansionSource,
|
||||
// activity based
|
||||
recentFollowingSimilarUsersSource: RecentFollowingSimilarUsersSource,
|
||||
recentEngagementSimilarUsersSource: RecentEngagementSimilarUsersSource,
|
||||
repeatedProfileVisitsSource: RepeatedProfileVisitsSource,
|
||||
// geo based
|
||||
popCountrySource: PopCountrySource,
|
||||
popGeohashSource: PopGeohashSource,
|
||||
popCountryBackFillSource: PopCountryBackFillSource,
|
||||
crowdSearchAccountsSource: CrowdSearchAccountsSource,
|
||||
topOrganicFollowsAccountsSource: TopOrganicFollowsAccountsSource,
|
||||
ppmiLocaleFollowSource: PPMILocaleFollowSource,
|
||||
baseStatsReceiver: StatsReceiver)
|
||||
extends CandidateSourceRegistry[ContentRecommenderRequest, CandidateUser] {
|
||||
|
||||
override val statsReceiver = baseStatsReceiver
|
||||
.scope("content_recommender_flow", "candidate_sources")
|
||||
|
||||
override val sources: Set[CandidateSource[ContentRecommenderRequest, CandidateUser]] = Seq(
|
||||
forwardPhoneBookSource,
|
||||
forwardEmailBookSource,
|
||||
reversePhoneBookSource,
|
||||
reverseEmailBookSource,
|
||||
offlineStrongTiePredictionSource,
|
||||
triangularLoopsSource,
|
||||
userUserGraphCandidateSource,
|
||||
realGraphOonSource,
|
||||
recentFollowingRecentFollowingExpansionSource,
|
||||
recentFollowingSimilarUsersSource,
|
||||
recentEngagementSimilarUsersSource,
|
||||
repeatedProfileVisitsSource,
|
||||
popCountrySource,
|
||||
popGeohashSource,
|
||||
popCountryBackFillSource,
|
||||
crowdSearchAccountsSource,
|
||||
topOrganicFollowsAccountsSource,
|
||||
ppmiLocaleFollowSource,
|
||||
).toSet
|
||||
}
|
||||
@ -0,0 +1,71 @@
|
||||
package com.twitter.follow_recommendations.flows.content_recommender_flow
|
||||
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardEmailBookSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardPhoneBookSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReverseEmailBookSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReversePhoneBookSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountryBackFillSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountrySource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.real_graph.RealGraphOonV2Source
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RepeatedProfileVisitsSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentFollowingSimilarUsersSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStrongTiePredictionSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.user_user_graph.UserUserGraphCandidateSource
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.socialgraph.RecentFollowingRecentFollowingExpansionSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsSource
|
||||
import com.twitter.timelines.configapi.Params
|
||||
|
||||
object ContentRecommenderFlowCandidateSourceWeights {
|
||||
|
||||
def getWeights(
|
||||
params: Params
|
||||
): Map[CandidateSourceIdentifier, Double] = {
|
||||
Map[CandidateSourceIdentifier, Double](
|
||||
// Social based
|
||||
UserUserGraphCandidateSource.Identifier -> params(
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.UserUserGraphSourceWeight),
|
||||
ForwardPhoneBookSource.Identifier -> params(
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.ForwardPhoneBookSourceWeight),
|
||||
ReversePhoneBookSource.Identifier -> params(
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.ReversePhoneBookSourceWeight),
|
||||
ForwardEmailBookSource.Identifier -> params(
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.ForwardEmailBookSourceWeight),
|
||||
ReverseEmailBookSource.Identifier -> params(
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.ReverseEmailBookSourceWeight),
|
||||
TriangularLoopsSource.Identifier -> params(
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.TriangularLoopsSourceWeight),
|
||||
OfflineStrongTiePredictionSource.Identifier -> params(
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.OfflineStrongTiePredictionSourceWeight),
|
||||
RecentFollowingRecentFollowingExpansionSource.Identifier -> params(
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.NewFollowingNewFollowingExpansionSourceWeight),
|
||||
RecentFollowingSimilarUsersSource.Identifier -> params(
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.NewFollowingSimilarUserSourceWeight),
|
||||
// Activity based
|
||||
RealGraphOonV2Source.Identifier -> params(
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.RealGraphOonSourceWeight),
|
||||
RecentEngagementSimilarUsersSource.Identifier -> params(
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.RecentEngagementSimilarUserSourceWeight),
|
||||
RepeatedProfileVisitsSource.Identifier -> params(
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.RepeatedProfileVisitsSourceWeight),
|
||||
// Geo based
|
||||
PopCountrySource.Identifier -> params(
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.PopCountrySourceWeight),
|
||||
PopGeohashSource.Identifier -> params(
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.PopGeohashSourceWeight),
|
||||
PopCountryBackFillSource.Identifier -> params(
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.PopCountryBackfillSourceWeight),
|
||||
PPMILocaleFollowSource.Identifier -> params(
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.PPMILocaleFollowSourceWeight),
|
||||
CrowdSearchAccountsSource.Identifier -> params(
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.CrowdSearchAccountSourceWeight),
|
||||
TopOrganicFollowsAccountsSource.Identifier -> params(
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.TopOrganicFollowsAccountsSourceWeight),
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,117 @@
|
||||
package com.twitter.follow_recommendations.flows.content_recommender_flow
|
||||
|
||||
import com.twitter.timelines.configapi.FSBoundedParam
|
||||
|
||||
object ContentRecommenderFlowCandidateSourceWeightsParams {
|
||||
// Social based
|
||||
case object ForwardPhoneBookSourceWeight
|
||||
extends FSBoundedParam[Double](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.ForwardPhoneBookSourceWeight,
|
||||
1d,
|
||||
0d,
|
||||
1000d)
|
||||
case object ForwardEmailBookSourceWeight
|
||||
extends FSBoundedParam[Double](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.ForwardEmailBookSourceWeight,
|
||||
1d,
|
||||
0d,
|
||||
1000d)
|
||||
case object ReversePhoneBookSourceWeight
|
||||
extends FSBoundedParam[Double](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.ReversePhoneBookSourceWeight,
|
||||
1d,
|
||||
0d,
|
||||
1000d)
|
||||
case object ReverseEmailBookSourceWeight
|
||||
extends FSBoundedParam[Double](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.ReverseEmailBookSourceWeight,
|
||||
1d,
|
||||
0d,
|
||||
1000d)
|
||||
case object OfflineStrongTiePredictionSourceWeight
|
||||
extends FSBoundedParam[Double](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.OfflineStrongTiePredictionSourceWeight,
|
||||
1d,
|
||||
0d,
|
||||
1000d)
|
||||
case object TriangularLoopsSourceWeight
|
||||
extends FSBoundedParam[Double](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.TriangularLoopsSourceWeight,
|
||||
1d,
|
||||
0d,
|
||||
1000d)
|
||||
case object UserUserGraphSourceWeight
|
||||
extends FSBoundedParam[Double](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.UserUserGraphSourceWeight,
|
||||
1d,
|
||||
0d,
|
||||
1000d)
|
||||
case object NewFollowingNewFollowingExpansionSourceWeight
|
||||
extends FSBoundedParam[Double](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.NewFollowingNewFollowingExpansionSourceWeight,
|
||||
1d,
|
||||
0d,
|
||||
1000d)
|
||||
// Activity based
|
||||
case object NewFollowingSimilarUserSourceWeight
|
||||
extends FSBoundedParam[Double](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.NewFollowingSimilarUserSourceWeight,
|
||||
1d,
|
||||
0d,
|
||||
1000d)
|
||||
case object RecentEngagementSimilarUserSourceWeight
|
||||
extends FSBoundedParam[Double](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.RecentEngagementSimilarUserSourceWeight,
|
||||
1d,
|
||||
0d,
|
||||
1000d)
|
||||
case object RepeatedProfileVisitsSourceWeight
|
||||
extends FSBoundedParam[Double](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.RepeatedProfileVisitsSourceWeight,
|
||||
1d,
|
||||
0d,
|
||||
1000d)
|
||||
case object RealGraphOonSourceWeight
|
||||
extends FSBoundedParam[Double](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.RealGraphOonSourceWeight,
|
||||
1d,
|
||||
0d,
|
||||
1000d)
|
||||
// Geo based
|
||||
case object PopCountrySourceWeight
|
||||
extends FSBoundedParam[Double](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.PopCountrySourceWeight,
|
||||
1d,
|
||||
0d,
|
||||
1000d)
|
||||
case object PopGeohashSourceWeight
|
||||
extends FSBoundedParam[Double](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.PopGeohashSourceWeight,
|
||||
1d,
|
||||
0d,
|
||||
1000d)
|
||||
case object PopCountryBackfillSourceWeight
|
||||
extends FSBoundedParam[Double](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.PopCountryBackfillSourceWeight,
|
||||
1d,
|
||||
0d,
|
||||
1000d)
|
||||
case object PPMILocaleFollowSourceWeight
|
||||
extends FSBoundedParam[Double](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.PPMILocaleFollowSourceWeight,
|
||||
1d,
|
||||
0d,
|
||||
1000d)
|
||||
case object TopOrganicFollowsAccountsSourceWeight
|
||||
extends FSBoundedParam[Double](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.TopOrganicFollowsAccountsSourceWeight,
|
||||
1d,
|
||||
0d,
|
||||
1000d)
|
||||
case object CrowdSearchAccountSourceWeight
|
||||
extends FSBoundedParam[Double](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.CrowdSearchAccountSourceWeight,
|
||||
1d,
|
||||
0d,
|
||||
1000d)
|
||||
}
|
||||
@ -0,0 +1,60 @@
|
||||
package com.twitter.follow_recommendations.flows.content_recommender_flow
|
||||
|
||||
import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig
|
||||
import com.twitter.timelines.configapi.FSBoundedParam
|
||||
import com.twitter.timelines.configapi.FSName
|
||||
import com.twitter.timelines.configapi.Param
|
||||
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class ContentRecommenderFlowFSConfig @Inject() () extends FeatureSwitchConfig {
|
||||
override val booleanFSParams: Seq[Param[Boolean] with FSName] =
|
||||
Seq(
|
||||
ContentRecommenderParams.IncludeActivityBasedCandidateSource,
|
||||
ContentRecommenderParams.IncludeSocialBasedCandidateSource,
|
||||
ContentRecommenderParams.IncludeGeoBasedCandidateSource,
|
||||
ContentRecommenderParams.IncludeHomeTimelineTweetRecsCandidateSource,
|
||||
ContentRecommenderParams.IncludeSocialProofEnforcedCandidateSource,
|
||||
ContentRecommenderParams.EnableRecentFollowingPredicate,
|
||||
ContentRecommenderParams.EnableGizmoduckPredicate,
|
||||
ContentRecommenderParams.EnableInactivePredicate,
|
||||
ContentRecommenderParams.EnableInvalidTargetCandidateRelationshipPredicate,
|
||||
ContentRecommenderParams.IncludeNewFollowingNewFollowingExpansionCandidateSource,
|
||||
ContentRecommenderParams.IncludeMoreGeoBasedCandidateSource,
|
||||
ContentRecommenderParams.TargetEligibility,
|
||||
ContentRecommenderParams.GetFollowersFromSgs,
|
||||
ContentRecommenderParams.EnableInvalidRelationshipPredicate,
|
||||
)
|
||||
|
||||
override val intFSParams: Seq[FSBoundedParam[Int]] =
|
||||
Seq(
|
||||
ContentRecommenderParams.ResultSizeParam,
|
||||
ContentRecommenderParams.BatchSizeParam,
|
||||
ContentRecommenderParams.FetchCandidateSourceBudgetInMillisecond,
|
||||
ContentRecommenderParams.RecentFollowingPredicateBudgetInMillisecond,
|
||||
)
|
||||
|
||||
override val doubleFSParams: Seq[FSBoundedParam[Double]] =
|
||||
Seq(
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.ForwardPhoneBookSourceWeight,
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.ForwardEmailBookSourceWeight,
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.ReversePhoneBookSourceWeight,
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.ReverseEmailBookSourceWeight,
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.OfflineStrongTiePredictionSourceWeight,
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.TriangularLoopsSourceWeight,
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.UserUserGraphSourceWeight,
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.NewFollowingNewFollowingExpansionSourceWeight,
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.NewFollowingSimilarUserSourceWeight,
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.RecentEngagementSimilarUserSourceWeight,
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.RepeatedProfileVisitsSourceWeight,
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.RealGraphOonSourceWeight,
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.PopCountrySourceWeight,
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.PopGeohashSourceWeight,
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.PopCountryBackfillSourceWeight,
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.PPMILocaleFollowSourceWeight,
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.TopOrganicFollowsAccountsSourceWeight,
|
||||
ContentRecommenderFlowCandidateSourceWeightsParams.CrowdSearchAccountSourceWeight,
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,70 @@
|
||||
package com.twitter.follow_recommendations.flows.content_recommender_flow
|
||||
|
||||
object ContentRecommenderFlowFeatureSwitchKeys {
|
||||
val TargetUserEligible = "content_recommender_flow_target_eligible"
|
||||
val ResultSize = "content_recommender_flow_result_size"
|
||||
val BatchSize = "content_recommender_flow_batch_size"
|
||||
val RecentFollowingPredicateBudgetInMillisecond =
|
||||
"content_recommender_flow_recent_following_predicate_budget_in_ms"
|
||||
val CandidateGenerationBudgetInMillisecond =
|
||||
"content_recommender_flow_candidate_generation_budget_in_ms"
|
||||
val EnableRecentFollowingPredicate = "content_recommender_flow_enable_recent_following_predicate"
|
||||
val EnableGizmoduckPredicate = "content_recommender_flow_enable_gizmoduck_predicate"
|
||||
val EnableInactivePredicate = "content_recommender_flow_enable_inactive_predicate"
|
||||
val EnableInvalidTargetCandidateRelationshipPredicate =
|
||||
"content_recommender_flow_enable_invalid_target_candidate_relationship_predicate"
|
||||
val IncludeActivityBasedCandidateSource =
|
||||
"content_recommender_flow_include_activity_based_candidate_source"
|
||||
val IncludeSocialBasedCandidateSource =
|
||||
"content_recommender_flow_include_social_based_candidate_source"
|
||||
val IncludeGeoBasedCandidateSource =
|
||||
"content_recommender_flow_include_geo_based_candidate_source"
|
||||
val IncludeHomeTimelineTweetRecsCandidateSource =
|
||||
"content_recommender_flow_include_home_timeline_tweet_recs_candidate_source"
|
||||
val IncludeSocialProofEnforcedCandidateSource =
|
||||
"content_recommender_flow_include_social_proof_enforced_candidate_source"
|
||||
val IncludeNewFollowingNewFollowingExpansionCandidateSource =
|
||||
"content_recommender_flow_include_new_following_new_following_expansion_candidate_source"
|
||||
val IncludeMoreGeoBasedCandidateSource =
|
||||
"content_recommender_flow_include_more_geo_based_candidate_source"
|
||||
val GetFollowersFromSgs = "content_recommender_flow_get_followers_from_sgs"
|
||||
val EnableInvalidRelationshipPredicate =
|
||||
"content_recommender_flow_enable_invalid_relationship_predicate"
|
||||
|
||||
// Candidate source weight param keys
|
||||
// Social based
|
||||
val ForwardPhoneBookSourceWeight =
|
||||
"content_recommender_flow_candidate_source_weight_forward_phone_book"
|
||||
val ForwardEmailBookSourceWeight =
|
||||
"content_recommender_flow_candidate_source_weight_forward_email_book"
|
||||
val ReversePhoneBookSourceWeight =
|
||||
"content_recommender_flow_candidate_source_weight_reverse_phone_book"
|
||||
val ReverseEmailBookSourceWeight =
|
||||
"content_recommender_flow_candidate_source_weight_reverse_email_book"
|
||||
val OfflineStrongTiePredictionSourceWeight =
|
||||
"content_recommender_flow_candidate_source_weight_offline_stp"
|
||||
val TriangularLoopsSourceWeight =
|
||||
"content_recommender_flow_candidate_source_weight_triangular_loops"
|
||||
val UserUserGraphSourceWeight = "content_recommender_flow_candidate_source_weight_user_user_graph"
|
||||
val NewFollowingNewFollowingExpansionSourceWeight =
|
||||
"content_recommender_flow_candidate_source_weight_new_following_new_following_expansion"
|
||||
// Activity based
|
||||
val NewFollowingSimilarUserSourceWeight =
|
||||
"content_recommender_flow_candidate_source_weight_new_following_similar_user"
|
||||
val RecentEngagementSimilarUserSourceWeight =
|
||||
"content_recommender_flow_candidate_source_weight_recent_engagement_similar_user"
|
||||
val RepeatedProfileVisitsSourceWeight =
|
||||
"content_recommender_flow_candidate_source_weight_repeated_profile_visits"
|
||||
val RealGraphOonSourceWeight = "content_recommender_flow_candidate_source_weight_real_graph_oon"
|
||||
// Geo based
|
||||
val PopCountrySourceWeight = "content_recommender_flow_candidate_source_weight_pop_country"
|
||||
val PopGeohashSourceWeight = "content_recommender_flow_candidate_source_weight_pop_geohash"
|
||||
val PopCountryBackfillSourceWeight =
|
||||
"content_recommender_flow_candidate_source_weight_pop_country_backfill"
|
||||
val PPMILocaleFollowSourceWeight =
|
||||
"content_recommender_flow_candidate_source_weight_ppmi_locale_follow"
|
||||
val TopOrganicFollowsAccountsSourceWeight =
|
||||
"content_recommender_flow_candidate_source_weight_top_organic_follow_account"
|
||||
val CrowdSearchAccountSourceWeight =
|
||||
"content_recommender_flow_candidate_source_weight_crowd_search_account"
|
||||
}
|
||||
@ -0,0 +1,85 @@
|
||||
package com.twitter.follow_recommendations.flows.content_recommender_flow
|
||||
|
||||
import com.twitter.timelines.configapi.FSBoundedParam
|
||||
import com.twitter.timelines.configapi.FSParam
|
||||
import com.twitter.timelines.configapi.Param
|
||||
|
||||
abstract class ContentRecommenderParams[A](default: A) extends Param[A](default) {
|
||||
override val statName: String = "content_recommender/" + this.getClass.getSimpleName
|
||||
}
|
||||
|
||||
object ContentRecommenderParams {
|
||||
|
||||
case object TargetEligibility
|
||||
extends FSParam[Boolean](ContentRecommenderFlowFeatureSwitchKeys.TargetUserEligible, true)
|
||||
|
||||
case object ResultSizeParam
|
||||
extends FSBoundedParam[Int](ContentRecommenderFlowFeatureSwitchKeys.ResultSize, 15, 1, 500)
|
||||
case object BatchSizeParam
|
||||
extends FSBoundedParam[Int](ContentRecommenderFlowFeatureSwitchKeys.BatchSize, 15, 1, 500)
|
||||
case object RecentFollowingPredicateBudgetInMillisecond
|
||||
extends FSBoundedParam[Int](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.RecentFollowingPredicateBudgetInMillisecond,
|
||||
8,
|
||||
1,
|
||||
50)
|
||||
case object FetchCandidateSourceBudgetInMillisecond
|
||||
extends FSBoundedParam[Int](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.CandidateGenerationBudgetInMillisecond,
|
||||
60,
|
||||
1,
|
||||
80)
|
||||
case object EnableRecentFollowingPredicate
|
||||
extends FSParam[Boolean](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.EnableRecentFollowingPredicate,
|
||||
true)
|
||||
case object EnableGizmoduckPredicate
|
||||
extends FSParam[Boolean](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.EnableGizmoduckPredicate,
|
||||
false)
|
||||
case object EnableInactivePredicate
|
||||
extends FSParam[Boolean](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.EnableInactivePredicate,
|
||||
false)
|
||||
case object EnableInvalidTargetCandidateRelationshipPredicate
|
||||
extends FSParam[Boolean](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.EnableInvalidTargetCandidateRelationshipPredicate,
|
||||
false)
|
||||
case object IncludeActivityBasedCandidateSource
|
||||
extends FSParam[Boolean](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.IncludeActivityBasedCandidateSource,
|
||||
true)
|
||||
case object IncludeSocialBasedCandidateSource
|
||||
extends FSParam[Boolean](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.IncludeSocialBasedCandidateSource,
|
||||
true)
|
||||
case object IncludeGeoBasedCandidateSource
|
||||
extends FSParam[Boolean](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.IncludeGeoBasedCandidateSource,
|
||||
true)
|
||||
case object IncludeHomeTimelineTweetRecsCandidateSource
|
||||
extends FSParam[Boolean](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.IncludeHomeTimelineTweetRecsCandidateSource,
|
||||
false)
|
||||
case object IncludeSocialProofEnforcedCandidateSource
|
||||
extends FSParam[Boolean](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.IncludeSocialProofEnforcedCandidateSource,
|
||||
false)
|
||||
case object IncludeNewFollowingNewFollowingExpansionCandidateSource
|
||||
extends FSParam[Boolean](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.IncludeNewFollowingNewFollowingExpansionCandidateSource,
|
||||
false)
|
||||
|
||||
case object IncludeMoreGeoBasedCandidateSource
|
||||
extends FSParam[Boolean](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.IncludeMoreGeoBasedCandidateSource,
|
||||
false)
|
||||
|
||||
case object GetFollowersFromSgs
|
||||
extends FSParam[Boolean](ContentRecommenderFlowFeatureSwitchKeys.GetFollowersFromSgs, false)
|
||||
|
||||
case object EnableInvalidRelationshipPredicate
|
||||
extends FSParam[Boolean](
|
||||
ContentRecommenderFlowFeatureSwitchKeys.EnableInvalidRelationshipPredicate,
|
||||
false)
|
||||
}
|
||||
@ -0,0 +1,45 @@
|
||||
package com.twitter.follow_recommendations.flows.content_recommender_flow
|
||||
|
||||
import com.twitter.core_workflows.user_model.thriftscala.UserState
|
||||
import com.twitter.follow_recommendations.common.models.DebugOptions
|
||||
import com.twitter.follow_recommendations.common.models.DisplayLocation
|
||||
import com.twitter.follow_recommendations.common.models.GeohashAndCountryCode
|
||||
import com.twitter.follow_recommendations.common.models.HasDebugOptions
|
||||
import com.twitter.follow_recommendations.common.models.HasDisplayLocation
|
||||
import com.twitter.follow_recommendations.common.models.HasExcludedUserIds
|
||||
import com.twitter.follow_recommendations.common.models.HasGeohashAndCountryCode
|
||||
import com.twitter.follow_recommendations.common.models.HasInvalidRelationshipUserIds
|
||||
import com.twitter.follow_recommendations.common.models.HasRecentFollowedByUserIds
|
||||
import com.twitter.follow_recommendations.common.models.HasRecentFollowedUserIds
|
||||
import com.twitter.follow_recommendations.common.models.HasUserState
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.ClientContext
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext
|
||||
import com.twitter.timelines.configapi.HasParams
|
||||
import com.twitter.timelines.configapi.Params
|
||||
|
||||
case class ContentRecommenderRequest(
|
||||
override val params: Params,
|
||||
override val clientContext: ClientContext,
|
||||
inputExcludeUserIds: Seq[Long],
|
||||
override val recentFollowedUserIds: Option[Seq[Long]],
|
||||
override val recentFollowedByUserIds: Option[Seq[Long]],
|
||||
override val invalidRelationshipUserIds: Option[Set[Long]],
|
||||
override val displayLocation: DisplayLocation,
|
||||
maxResults: Option[Int] = None,
|
||||
override val debugOptions: Option[DebugOptions] = None,
|
||||
override val geohashAndCountryCode: Option[GeohashAndCountryCode] = None,
|
||||
override val userState: Option[UserState] = None)
|
||||
extends HasParams
|
||||
with HasClientContext
|
||||
with HasDisplayLocation
|
||||
with HasDebugOptions
|
||||
with HasRecentFollowedUserIds
|
||||
with HasRecentFollowedByUserIds
|
||||
with HasInvalidRelationshipUserIds
|
||||
with HasExcludedUserIds
|
||||
with HasUserState
|
||||
with HasGeohashAndCountryCode {
|
||||
override val excludedUserIds: Seq[Long] = {
|
||||
inputExcludeUserIds ++ clientContext.userId.toSeq
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,121 @@
|
||||
package com.twitter.follow_recommendations.flows.content_recommender_flow
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.clients.geoduck.UserLocationFetcher
|
||||
import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient
|
||||
import com.twitter.follow_recommendations.common.clients.user_state.UserStateClient
|
||||
import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueOptionalWithStats
|
||||
import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueWithStats
|
||||
import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils.rescueWithStatsWithin
|
||||
import com.twitter.follow_recommendations.products.common.ProductRequest
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class ContentRecommenderRequestBuilder @Inject() (
|
||||
socialGraph: SocialGraphClient,
|
||||
userLocationFetcher: UserLocationFetcher,
|
||||
userStateClient: UserStateClient,
|
||||
statsReceiver: StatsReceiver) {
|
||||
|
||||
val stats: StatsReceiver = statsReceiver.scope("content_recommender_request_builder")
|
||||
val invalidRelationshipUsersStats: StatsReceiver = stats.scope("invalidRelationshipUserIds")
|
||||
private val invalidRelationshipUsersMaxSizeCounter =
|
||||
invalidRelationshipUsersStats.counter("maxSize")
|
||||
private val invalidRelationshipUsersNotMaxSizeCounter =
|
||||
invalidRelationshipUsersStats.counter("notMaxSize")
|
||||
|
||||
def build(req: ProductRequest): Stitch[ContentRecommenderRequest] = {
|
||||
val userStateStitch = Stitch
|
||||
.collect(req.recommendationRequest.clientContext.userId.map(userId =>
|
||||
userStateClient.getUserState(userId))).map(_.flatten)
|
||||
val recentFollowedUserIdsStitch =
|
||||
Stitch
|
||||
.collect(req.recommendationRequest.clientContext.userId.map { userId =>
|
||||
rescueWithStatsWithin(
|
||||
socialGraph.getRecentFollowedUserIds(userId),
|
||||
stats,
|
||||
"recentFollowedUserIds",
|
||||
req
|
||||
.params(
|
||||
ContentRecommenderParams.RecentFollowingPredicateBudgetInMillisecond).millisecond
|
||||
)
|
||||
})
|
||||
val recentFollowedByUserIdsStitch =
|
||||
if (req.params(ContentRecommenderParams.GetFollowersFromSgs)) {
|
||||
Stitch
|
||||
.collect(
|
||||
req.recommendationRequest.clientContext.userId.map(userId =>
|
||||
rescueWithStatsWithin(
|
||||
socialGraph.getRecentFollowedByUserIdsFromCachedColumn(userId),
|
||||
stats,
|
||||
"recentFollowedByUserIds",
|
||||
req
|
||||
.params(ContentRecommenderParams.RecentFollowingPredicateBudgetInMillisecond)
|
||||
.millisecond
|
||||
)))
|
||||
} else Stitch.None
|
||||
val invalidRelationshipUserIdsStitch: Stitch[Option[Seq[Long]]] =
|
||||
if (req.params(ContentRecommenderParams.EnableInvalidRelationshipPredicate)) {
|
||||
Stitch
|
||||
.collect(
|
||||
req.recommendationRequest.clientContext.userId.map { userId =>
|
||||
rescueWithStats(
|
||||
socialGraph
|
||||
.getInvalidRelationshipUserIdsFromCachedColumn(userId)
|
||||
.onSuccess(ids =>
|
||||
if (ids.size >= SocialGraphClient.MaxNumInvalidRelationship) {
|
||||
invalidRelationshipUsersMaxSizeCounter.incr()
|
||||
} else {
|
||||
invalidRelationshipUsersNotMaxSizeCounter.incr()
|
||||
}),
|
||||
stats,
|
||||
"invalidRelationshipUserIds"
|
||||
)
|
||||
}
|
||||
)
|
||||
} else {
|
||||
Stitch.None
|
||||
}
|
||||
val locationStitch =
|
||||
rescueOptionalWithStats(
|
||||
userLocationFetcher.getGeohashAndCountryCode(
|
||||
req.recommendationRequest.clientContext.userId,
|
||||
req.recommendationRequest.clientContext.ipAddress
|
||||
),
|
||||
stats,
|
||||
"userLocation"
|
||||
)
|
||||
Stitch
|
||||
.join(
|
||||
recentFollowedUserIdsStitch,
|
||||
recentFollowedByUserIdsStitch,
|
||||
invalidRelationshipUserIdsStitch,
|
||||
locationStitch,
|
||||
userStateStitch)
|
||||
.map {
|
||||
case (
|
||||
recentFollowedUserIds,
|
||||
recentFollowedByUserIds,
|
||||
invalidRelationshipUserIds,
|
||||
location,
|
||||
userState) =>
|
||||
ContentRecommenderRequest(
|
||||
req.params,
|
||||
req.recommendationRequest.clientContext,
|
||||
req.recommendationRequest.excludedIds.getOrElse(Nil),
|
||||
recentFollowedUserIds,
|
||||
recentFollowedByUserIds,
|
||||
invalidRelationshipUserIds.map(_.toSet),
|
||||
req.recommendationRequest.displayLocation,
|
||||
req.recommendationRequest.maxResults,
|
||||
req.recommendationRequest.debugParams.flatMap(_.debugOptions),
|
||||
location,
|
||||
userState
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,58 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/com/google/inject/extensions:guice-assistedinject",
|
||||
"3rdparty/jvm/net/codingwell:scala-guice",
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/addressbook",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/crowd_search_accounts",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/geo",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/real_graph",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/recent_engagement",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/salsa",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/sims_expansion",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/stp",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/top_organic_follows_accounts",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/triangular_loops",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/two_hop_random_walk",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/candidate_sources/user_user_graph",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/deepbirdv2",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/geoduck",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/impression_store",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/interests_service",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/clients/user_state",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/sources",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/dismiss",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/gizmoduck",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/health",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/sgs",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/predicates/user_activity",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/fatigue_ranker",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/first_n_ranker",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/interleave_ranker",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/ranking",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/ml_ranker/scoring",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/rankers/weighted_candidate_source_ranker",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/dedup",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/ranker_id",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/tracking_token",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/weighted_sampling",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/candidates",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/common",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/logging",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products/common",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/utils",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,103 @@
|
||||
package com.twitter.follow_recommendations.flows.post_nux_ml
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.base.CandidateSourceRegistry
|
||||
import com.twitter.follow_recommendations.common.base.EnrichedCandidateSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardEmailBookSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardPhoneBookSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReverseEmailBookSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReversePhoneBookSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountrySource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountryBackFillSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashQualityFollowSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.real_graph.RealGraphOonV2Source
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RecentEngagementNonDirectFollowSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RepeatedProfileVisitsSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.salsa.RecentEngagementDirectFollowSalsaExpansionSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.sims.LinearRegressionFollow2vecNearestNeighborsStore
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentFollowingSimilarUsersSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.stp.OnlineSTPSourceScorer
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStrongTiePredictionSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.user_user_graph.UserUserGraphCandidateSource
|
||||
import com.twitter.follow_recommendations.common.models.CandidateUser
|
||||
import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class PostNuxMlCandidateSourceRegistry @Inject() (
|
||||
crowdSearchAccountsCandidateSource: CrowdSearchAccountsSource,
|
||||
topOrganicFollowsAccountsSource: TopOrganicFollowsAccountsSource,
|
||||
linearRegressionfollow2vecNearestNeighborsStore: LinearRegressionFollow2vecNearestNeighborsStore,
|
||||
forwardEmailBookSource: ForwardEmailBookSource,
|
||||
forwardPhoneBookSource: ForwardPhoneBookSource,
|
||||
offlineStrongTiePredictionSource: OfflineStrongTiePredictionSource,
|
||||
onlineSTPSource: OnlineSTPSourceScorer,
|
||||
popCountrySource: PopCountrySource,
|
||||
popCountryBackFillSource: PopCountryBackFillSource,
|
||||
popGeohashSource: PopGeohashSource,
|
||||
recentEngagementDirectFollowSimilarUsersSource: RecentEngagementSimilarUsersSource,
|
||||
recentEngagementNonDirectFollowSource: RecentEngagementNonDirectFollowSource,
|
||||
recentEngagementDirectFollowSalsaExpansionSource: RecentEngagementDirectFollowSalsaExpansionSource,
|
||||
recentFollowingSimilarUsersSource: RecentFollowingSimilarUsersSource,
|
||||
realGraphOonV2Source: RealGraphOonV2Source,
|
||||
repeatedProfileVisitSource: RepeatedProfileVisitsSource,
|
||||
reverseEmailBookSource: ReverseEmailBookSource,
|
||||
reversePhoneBookSource: ReversePhoneBookSource,
|
||||
triangularLoopsSource: TriangularLoopsSource,
|
||||
userUserGraphCandidateSource: UserUserGraphCandidateSource,
|
||||
ppmiLocaleFollowSource: PPMILocaleFollowSource,
|
||||
popGeohashQualityFollowSource: PopGeohashQualityFollowSource,
|
||||
baseStatsReceiver: StatsReceiver,
|
||||
) extends CandidateSourceRegistry[PostNuxMlRequest, CandidateUser] {
|
||||
import EnrichedCandidateSource._
|
||||
|
||||
override val statsReceiver = baseStatsReceiver
|
||||
.scope("post_nux_ml_flow", "candidate_sources")
|
||||
|
||||
// sources primarily based on social graph signals
|
||||
private[this] val socialSources = Seq(
|
||||
linearRegressionfollow2vecNearestNeighborsStore.mapKeys[PostNuxMlRequest](
|
||||
_.getOptionalUserId.toSeq),
|
||||
forwardEmailBookSource,
|
||||
forwardPhoneBookSource,
|
||||
offlineStrongTiePredictionSource,
|
||||
onlineSTPSource,
|
||||
reverseEmailBookSource,
|
||||
reversePhoneBookSource,
|
||||
triangularLoopsSource,
|
||||
)
|
||||
|
||||
// sources primarily based on geo signals
|
||||
private[this] val geoSources = Seq(
|
||||
popCountrySource,
|
||||
popCountryBackFillSource,
|
||||
popGeohashSource,
|
||||
popGeohashQualityFollowSource,
|
||||
topOrganicFollowsAccountsSource,
|
||||
crowdSearchAccountsCandidateSource,
|
||||
ppmiLocaleFollowSource,
|
||||
)
|
||||
|
||||
// sources primarily based on recent activity signals
|
||||
private[this] val activitySources = Seq(
|
||||
repeatedProfileVisitSource,
|
||||
recentEngagementDirectFollowSalsaExpansionSource.mapKeys[PostNuxMlRequest](
|
||||
_.getOptionalUserId.toSeq),
|
||||
recentEngagementDirectFollowSimilarUsersSource,
|
||||
recentEngagementNonDirectFollowSource.mapKeys[PostNuxMlRequest](_.getOptionalUserId.toSeq),
|
||||
recentFollowingSimilarUsersSource,
|
||||
realGraphOonV2Source,
|
||||
userUserGraphCandidateSource,
|
||||
)
|
||||
|
||||
override val sources: Set[CandidateSource[PostNuxMlRequest, CandidateUser]] = (
|
||||
geoSources ++ socialSources ++ activitySources
|
||||
).toSet
|
||||
}
|
||||
@ -0,0 +1,177 @@
|
||||
package com.twitter.follow_recommendations.flows.post_nux_ml
|
||||
|
||||
import com.twitter.timelines.configapi.FSBoundedParam
|
||||
import com.twitter.timelines.configapi.Param
|
||||
|
||||
abstract class PostNuxMlCandidateSourceWeightParams[A](default: A) extends Param[A](default) {
|
||||
override val statName: String = "post_nux_ml/" + this.getClass.getSimpleName
|
||||
}
|
||||
|
||||
object PostNuxMlCandidateSourceWeightParams {
|
||||
|
||||
case object CandidateWeightCrowdSearch
|
||||
extends FSBoundedParam[Double](
|
||||
PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightCrowdSearch,
|
||||
1.0,
|
||||
0.0,
|
||||
1000.0
|
||||
)
|
||||
|
||||
case object CandidateWeightTopOrganicFollow
|
||||
extends FSBoundedParam[Double](
|
||||
PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightTopOrganicFollow,
|
||||
1.0,
|
||||
0.0,
|
||||
1000.0
|
||||
)
|
||||
case object CandidateWeightPPMILocaleFollow
|
||||
extends FSBoundedParam[Double](
|
||||
PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightPPMILocaleFollow,
|
||||
1.0,
|
||||
0.0,
|
||||
1000.0
|
||||
)
|
||||
|
||||
case object CandidateWeightForwardEmailBook
|
||||
extends FSBoundedParam[Double](
|
||||
PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightForwardEmailBook,
|
||||
1.0,
|
||||
0.0,
|
||||
1000.0
|
||||
)
|
||||
case object CandidateWeightForwardPhoneBook
|
||||
extends FSBoundedParam[Double](
|
||||
PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightForwardPhoneBook,
|
||||
1.0,
|
||||
0.0,
|
||||
1000.0
|
||||
)
|
||||
|
||||
case object CandidateWeightOfflineStrongTiePrediction
|
||||
extends FSBoundedParam[Double](
|
||||
PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightOfflineStrongTiePrediction,
|
||||
1.0,
|
||||
0.0,
|
||||
1000.0
|
||||
)
|
||||
case object CandidateWeightOnlineStp
|
||||
extends FSBoundedParam[Double](
|
||||
PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightOnlineStp,
|
||||
1.0,
|
||||
0.0,
|
||||
1000.0
|
||||
)
|
||||
case object CandidateWeightPopCountry
|
||||
extends FSBoundedParam[Double](
|
||||
PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightPopCountry,
|
||||
1.0,
|
||||
0.0,
|
||||
1000.0
|
||||
)
|
||||
case object CandidateWeightPopGeohash
|
||||
extends FSBoundedParam[Double](
|
||||
PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightPopGeohash,
|
||||
1.0,
|
||||
0.0,
|
||||
1000.0
|
||||
)
|
||||
case object CandidateWeightPopGeohashQualityFollow
|
||||
extends FSBoundedParam[Double](
|
||||
PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightPopGeohashQualityFollow,
|
||||
1.0,
|
||||
0.0,
|
||||
1000.0
|
||||
)
|
||||
case object CandidateWeightPopGeoBackfill
|
||||
extends FSBoundedParam[Double](
|
||||
PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightPopGeoBackfill,
|
||||
1,
|
||||
0.0,
|
||||
1000.0
|
||||
)
|
||||
case object CandidateWeightRecentFollowingSimilarUsers
|
||||
extends FSBoundedParam[Double](
|
||||
PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRecentFollowingSimilarUsers,
|
||||
1.0,
|
||||
0.0,
|
||||
1000.0
|
||||
)
|
||||
case object CandidateWeightRecentEngagementDirectFollowSalsaExpansion
|
||||
extends FSBoundedParam[Double](
|
||||
PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRecentEngagementDirectFollowSalsaExpansion,
|
||||
1.0,
|
||||
0.0,
|
||||
1000.0
|
||||
)
|
||||
case object CandidateWeightRecentEngagementNonDirectFollow
|
||||
extends FSBoundedParam[Double](
|
||||
PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRecentEngagementNonDirectFollow,
|
||||
1.0,
|
||||
0.0,
|
||||
1000.0
|
||||
)
|
||||
case object CandidateWeightRecentEngagementSimilarUsers
|
||||
extends FSBoundedParam[Double](
|
||||
PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRecentEngagementSimilarUsers,
|
||||
1.0,
|
||||
0.0,
|
||||
1000.0
|
||||
)
|
||||
case object CandidateWeightRepeatedProfileVisits
|
||||
extends FSBoundedParam[Double](
|
||||
PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRepeatedProfileVisits,
|
||||
1.0,
|
||||
0.0,
|
||||
1000.0
|
||||
)
|
||||
case object CandidateWeightFollow2vecNearestNeighbors
|
||||
extends FSBoundedParam[Double](
|
||||
PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightFollow2vecNearestNeighbors,
|
||||
1.0,
|
||||
0.0,
|
||||
1000.0
|
||||
)
|
||||
case object CandidateWeightReverseEmailBook
|
||||
extends FSBoundedParam[Double](
|
||||
PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightReverseEmailBook,
|
||||
1.0,
|
||||
0.0,
|
||||
1000.0
|
||||
)
|
||||
case object CandidateWeightReversePhoneBook
|
||||
extends FSBoundedParam[Double](
|
||||
PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightReversePhoneBook,
|
||||
1.0,
|
||||
0.0,
|
||||
1000.0
|
||||
)
|
||||
case object CandidateWeightTriangularLoops
|
||||
extends FSBoundedParam[Double](
|
||||
PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightTriangularLoops,
|
||||
1.0,
|
||||
0.0,
|
||||
1000.0
|
||||
)
|
||||
case object CandidateWeightTwoHopRandomWalk
|
||||
extends FSBoundedParam[Double](
|
||||
PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightTwoHopRandomWalk,
|
||||
1.0,
|
||||
0.0,
|
||||
1000.0
|
||||
)
|
||||
case object CandidateWeightUserUserGraph
|
||||
extends FSBoundedParam[Double](
|
||||
PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightUserUserGraph,
|
||||
1.0,
|
||||
0.0,
|
||||
1000.0
|
||||
)
|
||||
|
||||
case object CandidateWeightRealGraphOonV2
|
||||
extends FSBoundedParam[Double](
|
||||
PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys.CandidateWeightRealGraphOonV2,
|
||||
1.0,
|
||||
0.0,
|
||||
2000.0
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,193 @@
|
||||
package com.twitter.follow_recommendations.flows.post_nux_ml
|
||||
|
||||
import com.google.inject.Inject
|
||||
import com.google.inject.Singleton
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.base.IdentityRanker
|
||||
import com.twitter.follow_recommendations.common.base.IdentityTransform
|
||||
import com.twitter.follow_recommendations.common.base.Ranker
|
||||
import com.twitter.follow_recommendations.common.base.Transform
|
||||
import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature
|
||||
import com.twitter.follow_recommendations.common.models._
|
||||
import com.twitter.follow_recommendations.common.rankers.common.RankerId
|
||||
import com.twitter.follow_recommendations.common.rankers.fatigue_ranker.ImpressionBasedFatigueRanker
|
||||
import com.twitter.follow_recommendations.common.rankers.first_n_ranker.FirstNRanker
|
||||
import com.twitter.follow_recommendations.common.rankers.first_n_ranker.FirstNRankerParams
|
||||
import com.twitter.follow_recommendations.common.rankers.interleave_ranker.InterleaveRanker
|
||||
import com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking.HydrateFeaturesTransform
|
||||
import com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking.MlRanker
|
||||
import com.twitter.follow_recommendations.common.rankers.ml_ranker.ranking.MlRankerParams
|
||||
import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.WeightedCandidateSourceRanker
|
||||
import com.twitter.follow_recommendations.configapi.candidates.HydrateCandidateParamsTransform
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext
|
||||
import com.twitter.timelines.configapi.HasParams
|
||||
|
||||
/**
|
||||
* Used to build the combined ranker comprising 4 stages of ranking:
|
||||
* - weighted sampler
|
||||
* - truncating to the top N merged results for ranking
|
||||
* - ML ranker
|
||||
* - Interleaving ranker for producer-side experiments
|
||||
* - impression-based fatigueing
|
||||
*/
|
||||
@Singleton
|
||||
class PostNuxMlCombinedRankerBuilder[
|
||||
T <: HasParams with HasSimilarToContext with HasClientContext with HasExcludedUserIds with HasDisplayLocation with HasDebugOptions with HasPreFetchedFeature with HasDismissedUserIds with HasQualityFactor] @Inject() (
|
||||
firstNRanker: FirstNRanker[T],
|
||||
hydrateFeaturesTransform: HydrateFeaturesTransform[T],
|
||||
hydrateCandidateParamsTransform: HydrateCandidateParamsTransform[T],
|
||||
mlRanker: MlRanker[T],
|
||||
statsReceiver: StatsReceiver) {
|
||||
private[this] val stats: StatsReceiver = statsReceiver.scope("post_nux_ml_ranker")
|
||||
|
||||
// we construct each ranker independently and chain them together
|
||||
def build(
|
||||
request: T,
|
||||
candidateSourceWeights: Map[CandidateSourceIdentifier, Double]
|
||||
): Ranker[T, CandidateUser] = {
|
||||
val displayLocationStats = stats.scope(request.displayLocation.toString)
|
||||
val weightedRankerStats: StatsReceiver =
|
||||
displayLocationStats.scope("weighted_candidate_source_ranker")
|
||||
val firstNRankerStats: StatsReceiver =
|
||||
displayLocationStats.scope("first_n_ranker")
|
||||
val hydrateCandidateParamsStats =
|
||||
displayLocationStats.scope("hydrate_candidate_params")
|
||||
val fatigueRankerStats = displayLocationStats.scope("fatigue_ranker")
|
||||
val interleaveRankerStats =
|
||||
displayLocationStats.scope("interleave_ranker")
|
||||
val allRankersStats = displayLocationStats.scope("all_rankers")
|
||||
|
||||
// Checking if the heavy-ranker is an experimental model.
|
||||
// If it is, InterleaveRanker and candidate parameter hydration are disabled.
|
||||
// *NOTE* that consumer-side experiments should at any time take a small % of traffic, less
|
||||
// than 20% for instance, to leave enough room for producer experiments. Increasing bucket
|
||||
// size for producer experiments lead to other issues and is not a viable option for faster
|
||||
// experiments.
|
||||
val requestRankerId = request.params(MlRankerParams.RequestScorerIdParam)
|
||||
if (requestRankerId != RankerId.PostNuxProdRanker) {
|
||||
hydrateCandidateParamsStats.counter(s"disabled_by_${requestRankerId.toString}").incr()
|
||||
interleaveRankerStats.counter(s"disabled_by_${requestRankerId.toString}").incr()
|
||||
}
|
||||
|
||||
// weighted ranker that samples from the candidate sources
|
||||
val weightedRanker = WeightedCandidateSourceRanker
|
||||
.build[T](
|
||||
candidateSourceWeights,
|
||||
request.params(PostNuxMlParams.CandidateShuffler).shuffle(request.getRandomizationSeed),
|
||||
randomSeed = request.getRandomizationSeed
|
||||
).observe(weightedRankerStats)
|
||||
|
||||
// ranker that takes the first n results (ie truncates output) while merging duplicates
|
||||
val firstNRankerObs = firstNRanker.observe(firstNRankerStats)
|
||||
// either ML ranker that uses deepbirdv2 to score or no ranking
|
||||
val mainRanker: Ranker[T, CandidateUser] =
|
||||
buildMainRanker(request, requestRankerId == RankerId.PostNuxProdRanker, displayLocationStats)
|
||||
// fatigue ranker that uses wtf impressions to fatigue
|
||||
val fatigueRanker = buildFatigueRanker(request, fatigueRankerStats).observe(fatigueRankerStats)
|
||||
|
||||
// interleaveRanker combines rankings from several rankers and enforces candidates' ranks in
|
||||
// experiment buckets according to their assigned ranker model.
|
||||
val interleaveRanker =
|
||||
buildInterleaveRanker(
|
||||
request,
|
||||
requestRankerId == RankerId.PostNuxProdRanker,
|
||||
interleaveRankerStats)
|
||||
.observe(interleaveRankerStats)
|
||||
|
||||
weightedRanker
|
||||
.andThen(firstNRankerObs)
|
||||
.andThen(mainRanker)
|
||||
.andThen(fatigueRanker)
|
||||
.andThen(interleaveRanker)
|
||||
.observe(allRankersStats)
|
||||
}
|
||||
|
||||
def buildMainRanker(
|
||||
request: T,
|
||||
isMainRankerPostNuxProd: Boolean,
|
||||
displayLocationStats: StatsReceiver
|
||||
): Ranker[T, CandidateUser] = {
|
||||
|
||||
// note that we may be disabling heavy ranker for users not bucketed
|
||||
// (due to empty results from the new candidate source)
|
||||
// need a better solution in the future
|
||||
val mlRankerStats = displayLocationStats.scope("ml_ranker")
|
||||
val noMlRankerStats = displayLocationStats.scope("no_ml_ranker")
|
||||
val hydrateFeaturesStats =
|
||||
displayLocationStats.scope("hydrate_features")
|
||||
val hydrateCandidateParamsStats =
|
||||
displayLocationStats.scope("hydrate_candidate_params")
|
||||
val notHydrateCandidateParamsStats =
|
||||
displayLocationStats.scope("not_hydrate_candidate_params")
|
||||
val rankerStats = displayLocationStats.scope("ranker")
|
||||
val mlRankerDisabledByExperimentsCounter =
|
||||
mlRankerStats.counter("disabled_by_experiments")
|
||||
val mlRankerDisabledByQualityFactorCounter =
|
||||
mlRankerStats.counter("disabled_by_quality_factor")
|
||||
|
||||
val disabledByQualityFactor = request.qualityFactor
|
||||
.exists(_ <= request.params(PostNuxMlParams.TurnoffMLScorerQFThreshold))
|
||||
|
||||
if (disabledByQualityFactor)
|
||||
mlRankerDisabledByQualityFactorCounter.incr()
|
||||
|
||||
if (request.params(PostNuxMlParams.UseMlRanker) && !disabledByQualityFactor) {
|
||||
|
||||
val hydrateFeatures = hydrateFeaturesTransform
|
||||
.observe(hydrateFeaturesStats)
|
||||
|
||||
val optionalHydratedParamsTransform: Transform[T, CandidateUser] = {
|
||||
// We disable candidate parameter hydration for experimental heavy-ranker models.
|
||||
if (isMainRankerPostNuxProd &&
|
||||
request.params(PostNuxMlParams.EnableCandidateParamHydration)) {
|
||||
hydrateCandidateParamsTransform
|
||||
.observe(hydrateCandidateParamsStats)
|
||||
} else {
|
||||
new IdentityTransform[T, CandidateUser]()
|
||||
.observe(notHydrateCandidateParamsStats)
|
||||
}
|
||||
}
|
||||
val candidateSize = request.params(FirstNRankerParams.CandidatesToRank)
|
||||
Ranker
|
||||
.chain(
|
||||
hydrateFeatures.andThen(optionalHydratedParamsTransform),
|
||||
mlRanker.observe(mlRankerStats),
|
||||
)
|
||||
.within(
|
||||
request.params(PostNuxMlParams.MlRankerBudget),
|
||||
rankerStats.scope(s"n$candidateSize"))
|
||||
} else {
|
||||
new IdentityRanker[T, CandidateUser].observe(noMlRankerStats)
|
||||
}
|
||||
}
|
||||
|
||||
def buildInterleaveRanker(
|
||||
request: T,
|
||||
isMainRankerPostNuxProd: Boolean,
|
||||
interleaveRankerStats: StatsReceiver
|
||||
): Ranker[T, CandidateUser] = {
|
||||
// InterleaveRanker is enabled only for display locations powered by the PostNux heavy-ranker.
|
||||
if (request.params(PostNuxMlParams.EnableInterleaveRanker) &&
|
||||
// InterleaveRanker is disabled for requests with experimental heavy-rankers.
|
||||
isMainRankerPostNuxProd) {
|
||||
new InterleaveRanker[T](interleaveRankerStats)
|
||||
} else {
|
||||
new IdentityRanker[T, CandidateUser]()
|
||||
}
|
||||
}
|
||||
|
||||
def buildFatigueRanker(
|
||||
request: T,
|
||||
fatigueRankerStats: StatsReceiver
|
||||
): Ranker[T, CandidateUser] = {
|
||||
if (request.params(PostNuxMlParams.EnableFatigueRanker)) {
|
||||
ImpressionBasedFatigueRanker
|
||||
.build[T](
|
||||
fatigueRankerStats
|
||||
).within(request.params(PostNuxMlParams.FatigueRankerBudget), fatigueRankerStats)
|
||||
} else {
|
||||
new IdentityRanker[T, CandidateUser]()
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,304 @@
|
||||
package com.twitter.follow_recommendations.flows.post_nux_ml
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.base.EnrichedCandidateSource._
|
||||
import com.twitter.follow_recommendations.common.base._
|
||||
import com.twitter.follow_recommendations.common.models.CandidateUser
|
||||
import com.twitter.follow_recommendations.common.models.FilterReason
|
||||
import com.twitter.follow_recommendations.common.predicates.dismiss.DismissedCandidatePredicate
|
||||
import com.twitter.follow_recommendations.common.predicates.gizmoduck.GizmoduckPredicate
|
||||
import com.twitter.follow_recommendations.common.transforms.ranker_id.RandomRankerIdTransform
|
||||
import com.twitter.follow_recommendations.common.predicates.sgs.InvalidTargetCandidateRelationshipTypesPredicate
|
||||
import com.twitter.follow_recommendations.common.predicates.sgs.RecentFollowingPredicate
|
||||
import com.twitter.follow_recommendations.common.predicates.CandidateParamPredicate
|
||||
import com.twitter.follow_recommendations.common.predicates.CandidateSourceParamPredicate
|
||||
import com.twitter.follow_recommendations.common.predicates.CuratedCompetitorListPredicate
|
||||
import com.twitter.follow_recommendations.common.predicates.ExcludedUserIdPredicate
|
||||
import com.twitter.follow_recommendations.common.predicates.InactivePredicate
|
||||
import com.twitter.follow_recommendations.common.predicates.PreviouslyRecommendedUserIdsPredicate
|
||||
import com.twitter.follow_recommendations.common.predicates.user_activity.NonNearZeroUserActivityPredicate
|
||||
import com.twitter.follow_recommendations.common.transforms.dedup.DedupTransform
|
||||
import com.twitter.follow_recommendations.common.transforms.modify_social_proof.ModifySocialProofTransform
|
||||
import com.twitter.follow_recommendations.common.transforms.tracking_token.TrackingTokenTransform
|
||||
import com.twitter.follow_recommendations.common.transforms.weighted_sampling.SamplingTransform
|
||||
import com.twitter.follow_recommendations.configapi.candidates.CandidateUserParamsFactory
|
||||
import com.twitter.follow_recommendations.configapi.params.GlobalParams
|
||||
import com.twitter.follow_recommendations.configapi.params.GlobalParams.EnableGFSSocialProofTransform
|
||||
import com.twitter.follow_recommendations.utils.CandidateSourceHoldbackUtil
|
||||
import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier
|
||||
import com.twitter.timelines.configapi.Params
|
||||
import com.twitter.util.Duration
|
||||
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient
|
||||
import com.twitter.follow_recommendations.common.predicates.hss.HssPredicate
|
||||
import com.twitter.follow_recommendations.common.predicates.sgs.InvalidRelationshipPredicate
|
||||
import com.twitter.follow_recommendations.common.transforms.modify_social_proof.RemoveAccountProofTransform
|
||||
import com.twitter.follow_recommendations.logging.FrsLogger
|
||||
import com.twitter.follow_recommendations.models.RecommendationFlowData
|
||||
import com.twitter.follow_recommendations.utils.RecommendationFlowBaseSideEffectsUtil
|
||||
import com.twitter.product_mixer.core.model.common.identifier.RecommendationPipelineIdentifier
|
||||
import com.twitter.product_mixer.core.quality_factor.BoundsWithDefault
|
||||
import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactor
|
||||
import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactorConfig
|
||||
import com.twitter.product_mixer.core.quality_factor.LinearLatencyQualityFactorObserver
|
||||
import com.twitter.product_mixer.core.quality_factor.QualityFactorObserver
|
||||
import com.twitter.stitch.Stitch
|
||||
|
||||
/**
|
||||
* We use this flow for all post-nux display locations that would use a machine-learning-based-ranker
|
||||
* eg HTL, Sidebar, etc
|
||||
* Note that the RankedPostNuxFlow is used primarily for scribing/data collection, and doesn't
|
||||
* incorporate all of the other components in a flow (candidate source generation, predicates etc)
|
||||
*/
|
||||
@Singleton
|
||||
class PostNuxMlFlow @Inject() (
|
||||
postNuxMlCandidateSourceRegistry: PostNuxMlCandidateSourceRegistry,
|
||||
postNuxMlCombinedRankerBuilder: PostNuxMlCombinedRankerBuilder[PostNuxMlRequest],
|
||||
curatedCompetitorListPredicate: CuratedCompetitorListPredicate,
|
||||
gizmoduckPredicate: GizmoduckPredicate,
|
||||
sgsPredicate: InvalidTargetCandidateRelationshipTypesPredicate,
|
||||
hssPredicate: HssPredicate,
|
||||
invalidRelationshipPredicate: InvalidRelationshipPredicate,
|
||||
recentFollowingPredicate: RecentFollowingPredicate,
|
||||
nonNearZeroUserActivityPredicate: NonNearZeroUserActivityPredicate,
|
||||
inactivePredicate: InactivePredicate,
|
||||
dismissedCandidatePredicate: DismissedCandidatePredicate,
|
||||
previouslyRecommendedUserIdsPredicate: PreviouslyRecommendedUserIdsPredicate,
|
||||
modifySocialProofTransform: ModifySocialProofTransform,
|
||||
removeAccountProofTransform: RemoveAccountProofTransform,
|
||||
trackingTokenTransform: TrackingTokenTransform,
|
||||
randomRankerIdTransform: RandomRankerIdTransform,
|
||||
candidateParamsFactory: CandidateUserParamsFactory[PostNuxMlRequest],
|
||||
samplingTransform: SamplingTransform,
|
||||
frsLogger: FrsLogger,
|
||||
baseStatsReceiver: StatsReceiver)
|
||||
extends RecommendationFlow[PostNuxMlRequest, CandidateUser]
|
||||
with RecommendationFlowBaseSideEffectsUtil[PostNuxMlRequest, CandidateUser]
|
||||
with CandidateSourceHoldbackUtil {
|
||||
override protected val targetEligibility: Predicate[PostNuxMlRequest] =
|
||||
new ParamPredicate[PostNuxMlRequest](PostNuxMlParams.TargetEligibility)
|
||||
|
||||
override val statsReceiver: StatsReceiver = baseStatsReceiver.scope("post_nux_ml_flow")
|
||||
|
||||
override val qualityFactorObserver: Option[QualityFactorObserver] = {
|
||||
val config = LinearLatencyQualityFactorConfig(
|
||||
qualityFactorBounds =
|
||||
BoundsWithDefault(minInclusive = 0.1, maxInclusive = 1.0, default = 1.0),
|
||||
initialDelay = 60.seconds,
|
||||
targetLatency = 700.milliseconds,
|
||||
targetLatencyPercentile = 95.0,
|
||||
delta = 0.001
|
||||
)
|
||||
val qualityFactor = LinearLatencyQualityFactor(config)
|
||||
val observer = LinearLatencyQualityFactorObserver(qualityFactor)
|
||||
statsReceiver.provideGauge("quality_factor")(qualityFactor.currentValue.toFloat)
|
||||
Some(observer)
|
||||
}
|
||||
|
||||
override protected def updateTarget(request: PostNuxMlRequest): Stitch[PostNuxMlRequest] = {
|
||||
Stitch.value(
|
||||
request.copy(qualityFactor = qualityFactorObserver.map(_.qualityFactor.currentValue))
|
||||
)
|
||||
}
|
||||
|
||||
private[post_nux_ml] def getCandidateSourceIdentifiers(
|
||||
params: Params
|
||||
): Set[CandidateSourceIdentifier] = {
|
||||
PostNuxMlFlowCandidateSourceWeights.getWeights(params).keySet
|
||||
}
|
||||
|
||||
override protected def candidateSources(
|
||||
request: PostNuxMlRequest
|
||||
): Seq[CandidateSource[PostNuxMlRequest, CandidateUser]] = {
|
||||
val identifiers = getCandidateSourceIdentifiers(request.params)
|
||||
val selected: Set[CandidateSource[PostNuxMlRequest, CandidateUser]] =
|
||||
postNuxMlCandidateSourceRegistry.select(identifiers)
|
||||
val budget: Duration = request.params(PostNuxMlParams.FetchCandidateSourceBudget)
|
||||
filterCandidateSources(
|
||||
request,
|
||||
selected.map(c => c.failOpenWithin(budget, statsReceiver)).toSeq)
|
||||
}
|
||||
|
||||
override protected val preRankerCandidateFilter: Predicate[(PostNuxMlRequest, CandidateUser)] = {
|
||||
val stats = statsReceiver.scope("pre_ranker")
|
||||
|
||||
object excludeNearZeroUserPredicate
|
||||
extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)](
|
||||
nonNearZeroUserActivityPredicate,
|
||||
stats.scope("exclude_near_zero_predicate")
|
||||
) {
|
||||
override def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean =
|
||||
item._1.params(PostNuxMlParams.ExcludeNearZeroCandidates)
|
||||
}
|
||||
|
||||
object invalidRelationshipGatedPredicate
|
||||
extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)](
|
||||
invalidRelationshipPredicate,
|
||||
stats.scope("invalid_relationship_predicate")
|
||||
) {
|
||||
override def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean =
|
||||
item._1.params(PostNuxMlParams.EnableInvalidRelationshipPredicate)
|
||||
}
|
||||
|
||||
ExcludedUserIdPredicate
|
||||
.observe(stats.scope("exclude_user_id_predicate"))
|
||||
.andThen(
|
||||
recentFollowingPredicate.observe(stats.scope("recent_following_predicate"))
|
||||
)
|
||||
.andThen(
|
||||
dismissedCandidatePredicate.observe(stats.scope("dismissed_candidate_predicate"))
|
||||
)
|
||||
.andThen(
|
||||
previouslyRecommendedUserIdsPredicate.observe(
|
||||
stats.scope("previously_recommended_user_ids_predicate"))
|
||||
)
|
||||
.andThen(
|
||||
invalidRelationshipGatedPredicate.observe(stats.scope("invalid_relationship_predicate"))
|
||||
)
|
||||
.andThen(
|
||||
excludeNearZeroUserPredicate.observe(stats.scope("exclude_near_zero_user_state"))
|
||||
)
|
||||
.observe(stats.scope("overall_pre_ranker_candidate_filter"))
|
||||
}
|
||||
|
||||
override protected def selectRanker(
|
||||
request: PostNuxMlRequest
|
||||
): Ranker[PostNuxMlRequest, CandidateUser] = {
|
||||
postNuxMlCombinedRankerBuilder.build(
|
||||
request,
|
||||
PostNuxMlFlowCandidateSourceWeights.getWeights(request.params))
|
||||
}
|
||||
|
||||
override protected val postRankerTransform: Transform[PostNuxMlRequest, CandidateUser] = {
|
||||
new DedupTransform[PostNuxMlRequest, CandidateUser]
|
||||
.observe(statsReceiver.scope("dedupping"))
|
||||
.andThen(
|
||||
samplingTransform
|
||||
.gated(PostNuxMlParams.SamplingTransformEnabled)
|
||||
.observe(statsReceiver.scope("samplingtransform")))
|
||||
}
|
||||
|
||||
override protected val validateCandidates: Predicate[(PostNuxMlRequest, CandidateUser)] = {
|
||||
val stats = statsReceiver.scope("validate_candidates")
|
||||
val competitorPredicate =
|
||||
curatedCompetitorListPredicate.map[(PostNuxMlRequest, CandidateUser)](_._2)
|
||||
|
||||
val producerHoldbackPredicate = new CandidateParamPredicate[CandidateUser](
|
||||
GlobalParams.KeepUserCandidate,
|
||||
FilterReason.CandidateSideHoldback
|
||||
).map[(PostNuxMlRequest, CandidateUser)] {
|
||||
case (request, user) => candidateParamsFactory(user, request)
|
||||
}
|
||||
val pymkProducerHoldbackPredicate = new CandidateSourceParamPredicate(
|
||||
GlobalParams.KeepSocialUserCandidate,
|
||||
FilterReason.CandidateSideHoldback,
|
||||
CandidateSourceHoldbackUtil.SocialCandidateSourceIds
|
||||
).map[(PostNuxMlRequest, CandidateUser)] {
|
||||
case (request, user) => candidateParamsFactory(user, request)
|
||||
}
|
||||
val sgsPredicateStats = stats.scope("sgs_predicate")
|
||||
object sgsGatedPredicate
|
||||
extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)](
|
||||
sgsPredicate.observe(sgsPredicateStats),
|
||||
sgsPredicateStats
|
||||
) {
|
||||
|
||||
/**
|
||||
* When SGS predicate is turned off, only query SGS exists API for (user, candidate, relationship)
|
||||
* when the user's number of invalid relationships exceeds the threshold during request
|
||||
* building step. This is to minimize load to SGS and underlying Flock DB.
|
||||
*/
|
||||
override def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean =
|
||||
item._1.params(PostNuxMlParams.EnableSGSPredicate) ||
|
||||
SocialGraphClient.enablePostRankerSgsPredicate(
|
||||
item._1.invalidRelationshipUserIds.getOrElse(Set.empty).size)
|
||||
}
|
||||
|
||||
val hssPredicateStats = stats.scope("hss_predicate")
|
||||
object hssGatedPredicate
|
||||
extends GatedPredicateBase[(PostNuxMlRequest, CandidateUser)](
|
||||
hssPredicate.observe(hssPredicateStats),
|
||||
hssPredicateStats
|
||||
) {
|
||||
override def gate(item: (PostNuxMlRequest, CandidateUser)): Boolean =
|
||||
item._1.params(PostNuxMlParams.EnableHssPredicate)
|
||||
}
|
||||
|
||||
Predicate
|
||||
.andConcurrently[(PostNuxMlRequest, CandidateUser)](
|
||||
Seq(
|
||||
competitorPredicate.observe(stats.scope("curated_competitor_predicate")),
|
||||
gizmoduckPredicate.observe(stats.scope("gizmoduck_predicate")),
|
||||
sgsGatedPredicate,
|
||||
hssGatedPredicate,
|
||||
inactivePredicate.observe(stats.scope("inactive_predicate")),
|
||||
)
|
||||
)
|
||||
// to avoid dilutions, we need to apply the receiver holdback predicates at the very last step
|
||||
.andThen(pymkProducerHoldbackPredicate.observe(stats.scope("pymk_receiver_side_holdback")))
|
||||
.andThen(producerHoldbackPredicate.observe(stats.scope("receiver_side_holdback")))
|
||||
.observe(stats.scope("overall_validate_candidates"))
|
||||
}
|
||||
|
||||
override protected val transformResults: Transform[PostNuxMlRequest, CandidateUser] = {
|
||||
modifySocialProofTransform
|
||||
.gated(EnableGFSSocialProofTransform)
|
||||
.andThen(trackingTokenTransform)
|
||||
.andThen(randomRankerIdTransform.gated(PostNuxMlParams.LogRandomRankerId))
|
||||
.andThen(removeAccountProofTransform.gated(PostNuxMlParams.EnableRemoveAccountProofTransform))
|
||||
}
|
||||
|
||||
override protected def resultsConfig(request: PostNuxMlRequest): RecommendationResultsConfig = {
|
||||
RecommendationResultsConfig(
|
||||
request.maxResults.getOrElse(request.params(PostNuxMlParams.ResultSizeParam)),
|
||||
request.params(PostNuxMlParams.BatchSizeParam)
|
||||
)
|
||||
}
|
||||
|
||||
override def applySideEffects(
|
||||
target: PostNuxMlRequest,
|
||||
candidateSources: Seq[CandidateSource[PostNuxMlRequest, CandidateUser]],
|
||||
candidatesFromCandidateSources: Seq[CandidateUser],
|
||||
mergedCandidates: Seq[CandidateUser],
|
||||
filteredCandidates: Seq[CandidateUser],
|
||||
rankedCandidates: Seq[CandidateUser],
|
||||
transformedCandidates: Seq[CandidateUser],
|
||||
truncatedCandidates: Seq[CandidateUser],
|
||||
results: Seq[CandidateUser]
|
||||
): Stitch[Unit] = {
|
||||
frsLogger.logRecommendationFlowData[PostNuxMlRequest](
|
||||
target,
|
||||
RecommendationFlowData[PostNuxMlRequest](
|
||||
target,
|
||||
PostNuxMlFlow.identifier,
|
||||
candidateSources,
|
||||
candidatesFromCandidateSources,
|
||||
mergedCandidates,
|
||||
filteredCandidates,
|
||||
rankedCandidates,
|
||||
transformedCandidates,
|
||||
truncatedCandidates,
|
||||
results
|
||||
)
|
||||
)
|
||||
super.applySideEffects(
|
||||
target,
|
||||
candidateSources,
|
||||
candidatesFromCandidateSources,
|
||||
mergedCandidates,
|
||||
filteredCandidates,
|
||||
rankedCandidates,
|
||||
transformedCandidates,
|
||||
truncatedCandidates,
|
||||
results
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
object PostNuxMlFlow {
|
||||
val identifier = RecommendationPipelineIdentifier("PostNuxMlFlow")
|
||||
}
|
||||
@ -0,0 +1,68 @@
|
||||
package com.twitter.follow_recommendations.flows.post_nux_ml
|
||||
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardEmailBookSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ForwardPhoneBookSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReverseEmailBookSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.addressbook.ReversePhoneBookSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.crowd_search_accounts.CrowdSearchAccountsSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountryBackFillSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.geo.PopCountrySource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashQualityFollowSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.geo.PopGeohashSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.ppmi_locale_follow.PPMILocaleFollowSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.real_graph.RealGraphOonV2Source
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RecentEngagementNonDirectFollowSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.recent_engagement.RepeatedProfileVisitsSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.salsa.RecentEngagementDirectFollowSalsaExpansionSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentEngagementSimilarUsersSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.sims_expansion.RecentFollowingSimilarUsersSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.sims.Follow2vecNearestNeighborsStore
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.stp.BaseOnlineSTPSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.stp.OfflineStrongTiePredictionSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.top_organic_follows_accounts.TopOrganicFollowsAccountsSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.triangular_loops.TriangularLoopsSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.two_hop_random_walk.TwoHopRandomWalkSource
|
||||
import com.twitter.follow_recommendations.common.candidate_sources.user_user_graph.UserUserGraphCandidateSource
|
||||
import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlCandidateSourceWeightParams._
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier
|
||||
import com.twitter.timelines.configapi.Params
|
||||
|
||||
object PostNuxMlFlowCandidateSourceWeights {
|
||||
|
||||
def getWeights(params: Params): Map[CandidateSourceIdentifier, Double] = {
|
||||
Map[CandidateSourceIdentifier, Double](
|
||||
// Social based
|
||||
PPMILocaleFollowSource.Identifier -> params(CandidateWeightPPMILocaleFollow),
|
||||
Follow2vecNearestNeighborsStore.IdentifierF2vLinearRegression -> params(
|
||||
CandidateWeightFollow2vecNearestNeighbors),
|
||||
RecentFollowingSimilarUsersSource.Identifier -> params(
|
||||
CandidateWeightRecentFollowingSimilarUsers),
|
||||
BaseOnlineSTPSource.Identifier -> params(CandidateWeightOnlineStp),
|
||||
OfflineStrongTiePredictionSource.Identifier -> params(
|
||||
CandidateWeightOfflineStrongTiePrediction),
|
||||
ForwardEmailBookSource.Identifier -> params(CandidateWeightForwardEmailBook),
|
||||
ForwardPhoneBookSource.Identifier -> params(CandidateWeightForwardPhoneBook),
|
||||
ReverseEmailBookSource.Identifier -> params(CandidateWeightReverseEmailBook),
|
||||
ReversePhoneBookSource.Identifier -> params(CandidateWeightReversePhoneBook),
|
||||
TriangularLoopsSource.Identifier -> params(CandidateWeightTriangularLoops),
|
||||
TwoHopRandomWalkSource.Identifier -> params(CandidateWeightTwoHopRandomWalk),
|
||||
UserUserGraphCandidateSource.Identifier -> params(CandidateWeightUserUserGraph),
|
||||
// Geo based
|
||||
PopCountrySource.Identifier -> params(CandidateWeightPopCountry),
|
||||
PopCountryBackFillSource.Identifier -> params(CandidateWeightPopGeoBackfill),
|
||||
PopGeohashSource.Identifier -> params(CandidateWeightPopGeohash),
|
||||
PopGeohashQualityFollowSource.Identifier -> params(CandidateWeightPopGeohashQualityFollow),
|
||||
CrowdSearchAccountsSource.Identifier -> params(CandidateWeightCrowdSearch),
|
||||
TopOrganicFollowsAccountsSource.Identifier -> params(CandidateWeightTopOrganicFollow),
|
||||
// Engagement based
|
||||
RealGraphOonV2Source.Identifier -> params(CandidateWeightRealGraphOonV2),
|
||||
RecentEngagementNonDirectFollowSource.Identifier -> params(
|
||||
CandidateWeightRecentEngagementNonDirectFollow),
|
||||
RecentEngagementSimilarUsersSource.Identifier -> params(
|
||||
CandidateWeightRecentEngagementSimilarUsers),
|
||||
RepeatedProfileVisitsSource.Identifier -> params(CandidateWeightRepeatedProfileVisits),
|
||||
RecentEngagementDirectFollowSalsaExpansionSource.Identifier -> params(
|
||||
CandidateWeightRecentEngagementDirectFollowSalsaExpansion),
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,46 @@
|
||||
package com.twitter.follow_recommendations.flows.post_nux_ml
|
||||
|
||||
object PostNuxMlFlowCandidateSourceWeightsFeatureSwitchKeys {
|
||||
val CandidateWeightCrowdSearch = "post_nux_ml_flow_candidate_source_weights_user_crowd_search"
|
||||
val CandidateWeightTopOrganicFollow =
|
||||
"post_nux_ml_flow_candidate_source_weights_top_organic_follow"
|
||||
val CandidateWeightPPMILocaleFollow =
|
||||
"post_nux_ml_flow_candidate_source_weights_user_ppmi_locale_follow"
|
||||
val CandidateWeightForwardEmailBook =
|
||||
"post_nux_ml_flow_candidate_source_weights_user_forward_email_book"
|
||||
val CandidateWeightForwardPhoneBook =
|
||||
"post_nux_ml_flow_candidate_source_weights_user_forward_phone_book"
|
||||
val CandidateWeightOfflineStrongTiePrediction =
|
||||
"post_nux_ml_flow_candidate_source_weights_user_offline_strong_tie_prediction"
|
||||
val CandidateWeightOnlineStp = "post_nux_ml_flow_candidate_source_weights_user_online_stp"
|
||||
val CandidateWeightPopCountry = "post_nux_ml_flow_candidate_source_weights_user_pop_country"
|
||||
val CandidateWeightPopGeohash = "post_nux_ml_flow_candidate_source_weights_user_pop_geohash"
|
||||
val CandidateWeightPopGeohashQualityFollow =
|
||||
"post_nux_ml_flow_candidate_source_weights_user_pop_geohash_quality_follow"
|
||||
val CandidateWeightPopGeoBackfill =
|
||||
"post_nux_ml_flow_candidate_source_weights_user_pop_geo_backfill"
|
||||
val CandidateWeightRecentFollowingSimilarUsers =
|
||||
"post_nux_ml_flow_candidate_source_weights_user_recent_following_similar_users"
|
||||
val CandidateWeightRecentEngagementDirectFollowSalsaExpansion =
|
||||
"post_nux_ml_flow_candidate_source_weights_user_recent_engagement_direct_follow_salsa_expansion"
|
||||
val CandidateWeightRecentEngagementNonDirectFollow =
|
||||
"post_nux_ml_flow_candidate_source_weights_user_recent_engagement_non_direct_follow"
|
||||
val CandidateWeightRecentEngagementSimilarUsers =
|
||||
"post_nux_ml_flow_candidate_source_weights_user_recent_engagement_similar_users"
|
||||
val CandidateWeightRepeatedProfileVisits =
|
||||
"post_nux_ml_flow_candidate_source_weights_user_repeated_profile_visits"
|
||||
val CandidateWeightFollow2vecNearestNeighbors =
|
||||
"post_nux_ml_flow_candidate_source_weights_user_follow2vec_nearest_neighbors"
|
||||
val CandidateWeightReverseEmailBook =
|
||||
"post_nux_ml_flow_candidate_source_weights_user_reverse_email_book"
|
||||
val CandidateWeightReversePhoneBook =
|
||||
"post_nux_ml_flow_candidate_source_weights_user_reverse_phone_book"
|
||||
val CandidateWeightTriangularLoops =
|
||||
"post_nux_ml_flow_candidate_source_weights_user_triangular_loops"
|
||||
val CandidateWeightTwoHopRandomWalk =
|
||||
"post_nux_ml_flow_candidate_source_weights_user_two_hop_random_walk"
|
||||
val CandidateWeightUserUserGraph =
|
||||
"post_nux_ml_flow_candidate_source_weights_user_user_user_graph"
|
||||
val CandidateWeightRealGraphOonV2 =
|
||||
"post_nux_ml_flow_candidate_source_weights_user_real_graph_oon_v2"
|
||||
}
|
||||
@ -0,0 +1,80 @@
|
||||
package com.twitter.follow_recommendations.flows.post_nux_ml
|
||||
|
||||
import com.twitter.follow_recommendations.common.models.CandidateUser
|
||||
import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.NoShuffle
|
||||
import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.RandomShuffler
|
||||
import com.twitter.follow_recommendations.configapi.common.FeatureSwitchConfig
|
||||
import com.twitter.timelines.configapi.FSBoundedParam
|
||||
import com.twitter.timelines.configapi.FSName
|
||||
import com.twitter.timelines.configapi.HasDurationConversion
|
||||
import com.twitter.timelines.configapi.Param
|
||||
import com.twitter.util.Duration
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class PostNuxMlFlowFSConfig @Inject() () extends FeatureSwitchConfig {
|
||||
override val booleanFSParams: Seq[Param[Boolean] with FSName] = Seq(
|
||||
PostNuxMlParams.OnlineSTPEnabled,
|
||||
PostNuxMlParams.SamplingTransformEnabled,
|
||||
PostNuxMlParams.Follow2VecLinearRegressionEnabled,
|
||||
PostNuxMlParams.UseMlRanker,
|
||||
PostNuxMlParams.EnableCandidateParamHydration,
|
||||
PostNuxMlParams.EnableInterleaveRanker,
|
||||
PostNuxMlParams.EnableAdhocRanker,
|
||||
PostNuxMlParams.ExcludeNearZeroCandidates,
|
||||
PostNuxMlParams.IncludeRepeatedProfileVisitsCandidateSource,
|
||||
PostNuxMlParams.EnableInterestsOptOutPredicate,
|
||||
PostNuxMlParams.EnableSGSPredicate,
|
||||
PostNuxMlParams.EnableInvalidRelationshipPredicate,
|
||||
PostNuxMlParams.EnableRemoveAccountProofTransform,
|
||||
PostNuxMlParams.EnablePPMILocaleFollowSourceInPostNux,
|
||||
PostNuxMlParams.EnableRealGraphOonV2,
|
||||
PostNuxMlParams.GetFollowersFromSgs,
|
||||
PostNuxMlRequestBuilderParams.EnableInvalidRelationshipPredicate
|
||||
)
|
||||
|
||||
override val doubleFSParams: Seq[FSBoundedParam[Double]] = Seq(
|
||||
PostNuxMlCandidateSourceWeightParams.CandidateWeightCrowdSearch,
|
||||
PostNuxMlCandidateSourceWeightParams.CandidateWeightTopOrganicFollow,
|
||||
PostNuxMlCandidateSourceWeightParams.CandidateWeightPPMILocaleFollow,
|
||||
PostNuxMlCandidateSourceWeightParams.CandidateWeightForwardEmailBook,
|
||||
PostNuxMlCandidateSourceWeightParams.CandidateWeightForwardPhoneBook,
|
||||
PostNuxMlCandidateSourceWeightParams.CandidateWeightOfflineStrongTiePrediction,
|
||||
PostNuxMlCandidateSourceWeightParams.CandidateWeightOnlineStp,
|
||||
PostNuxMlCandidateSourceWeightParams.CandidateWeightPopCountry,
|
||||
PostNuxMlCandidateSourceWeightParams.CandidateWeightPopGeohash,
|
||||
PostNuxMlCandidateSourceWeightParams.CandidateWeightPopGeohashQualityFollow,
|
||||
PostNuxMlCandidateSourceWeightParams.CandidateWeightPopGeoBackfill,
|
||||
PostNuxMlCandidateSourceWeightParams.CandidateWeightRecentFollowingSimilarUsers,
|
||||
PostNuxMlCandidateSourceWeightParams.CandidateWeightRecentEngagementDirectFollowSalsaExpansion,
|
||||
PostNuxMlCandidateSourceWeightParams.CandidateWeightRecentEngagementNonDirectFollow,
|
||||
PostNuxMlCandidateSourceWeightParams.CandidateWeightRecentEngagementSimilarUsers,
|
||||
PostNuxMlCandidateSourceWeightParams.CandidateWeightRepeatedProfileVisits,
|
||||
PostNuxMlCandidateSourceWeightParams.CandidateWeightFollow2vecNearestNeighbors,
|
||||
PostNuxMlCandidateSourceWeightParams.CandidateWeightReverseEmailBook,
|
||||
PostNuxMlCandidateSourceWeightParams.CandidateWeightReversePhoneBook,
|
||||
PostNuxMlCandidateSourceWeightParams.CandidateWeightTriangularLoops,
|
||||
PostNuxMlCandidateSourceWeightParams.CandidateWeightTwoHopRandomWalk,
|
||||
PostNuxMlCandidateSourceWeightParams.CandidateWeightUserUserGraph,
|
||||
PostNuxMlCandidateSourceWeightParams.CandidateWeightRealGraphOonV2,
|
||||
PostNuxMlParams.TurnoffMLScorerQFThreshold
|
||||
)
|
||||
|
||||
override val durationFSParams: Seq[FSBoundedParam[Duration] with HasDurationConversion] = Seq(
|
||||
PostNuxMlParams.MlRankerBudget,
|
||||
PostNuxMlRequestBuilderParams.TopicIdFetchBudget,
|
||||
PostNuxMlRequestBuilderParams.DismissedIdScanBudget,
|
||||
PostNuxMlRequestBuilderParams.WTFImpressionsScanBudget
|
||||
)
|
||||
|
||||
override val gatedOverridesMap = Map(
|
||||
PostNuxMlFlowFeatureSwitchKeys.EnableRandomDataCollection -> Seq(
|
||||
PostNuxMlParams.CandidateShuffler := new RandomShuffler[CandidateUser],
|
||||
PostNuxMlParams.LogRandomRankerId := true
|
||||
),
|
||||
PostNuxMlFlowFeatureSwitchKeys.EnableNoShuffler -> Seq(
|
||||
PostNuxMlParams.CandidateShuffler := new NoShuffle[CandidateUser]
|
||||
),
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,27 @@
|
||||
package com.twitter.follow_recommendations.flows.post_nux_ml
|
||||
|
||||
object PostNuxMlFlowFeatureSwitchKeys {
|
||||
val UseMlRanker = "post_nux_ml_flow_use_ml_ranker"
|
||||
val EnableCandidateParamHydration = "post_nux_ml_flow_enable_candidate_param_hydration"
|
||||
val OnlineSTPEnabled = "post_nux_ml_flow_online_stp_source_enabled"
|
||||
val Follow2VecLinearRegressionEnabled = "post_nux_ml_flow_follow_to_vec_lr_source_enabled"
|
||||
val EnableRandomDataCollection = "post_nux_ml_flow_random_data_collection_enabled"
|
||||
val EnableAdhocRanker = "post_nux_ml_flow_adhoc_ranker_enabled"
|
||||
val EnableFatigueRanker = "post_nux_ml_flow_fatigue_ranker_enabled"
|
||||
val EnableInterleaveRanker = "post_nux_ml_flow_interleave_ranker_enabled"
|
||||
val IncludeRepeatedProfileVisitsCandidateSource =
|
||||
"post_nux_ml_flow_include_repeated_profile_visits_candidate_source"
|
||||
val MLRankerBudget = "post_nux_ml_flow_ml_ranker_budget_millis"
|
||||
val EnableNoShuffler = "post_nux_ml_flow_no_shuffler"
|
||||
val SamplingTransformEnabled = "post_nux_ml_flow_sampling_transform_enabled"
|
||||
val ExcludeNearZeroCandidates = "post_nux_ml_flow_exclude_near_zero_candidates"
|
||||
val EnableInterestsOptOutPredicate = "post_nux_ml_flow_enable_interests_opt_out_predicate"
|
||||
val EnableRemoveAccountProofTransform = "post_nux_ml_flow_enable_remove_account_proof_transform"
|
||||
val EnablePPMILocaleFollowSourceInPostNux = "post_nux_ml_flow_enable_ppmilocale_follow_source"
|
||||
val EnableInvalidRelationshipPredicate = "post_nux_ml_flow_enable_invalid_relationship_predicate"
|
||||
val EnableRealGraphOonV2 = "post_nux_ml_flow_enable_real_graph_oon_v2"
|
||||
val EnableSGSPredicate = "post_nux_ml_flow_enable_sgs_predicate"
|
||||
val EnableHssPredicate = "post_nux_ml_flow_enable_hss_predicate"
|
||||
val GetFollowersFromSgs = "post_nux_ml_flow_get_followers_from_sgs"
|
||||
val TurnOffMLScorerQFThreshold = "post_nux_ml_flow_turn_off_ml_scorer_threhsold"
|
||||
}
|
||||
@ -0,0 +1,133 @@
|
||||
package com.twitter.follow_recommendations.flows.post_nux_ml
|
||||
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.follow_recommendations.common.models.CandidateUser
|
||||
import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.CandidateShuffler
|
||||
import com.twitter.follow_recommendations.common.rankers.weighted_candidate_source_ranker.ExponentialShuffler
|
||||
import com.twitter.timelines.configapi.DurationConversion
|
||||
import com.twitter.timelines.configapi.FSBoundedParam
|
||||
import com.twitter.timelines.configapi.FSParam
|
||||
import com.twitter.timelines.configapi.HasDurationConversion
|
||||
import com.twitter.timelines.configapi.Param
|
||||
import com.twitter.util.Duration
|
||||
|
||||
abstract class PostNuxMlParams[A](default: A) extends Param[A](default) {
|
||||
override val statName: String = "post_nux_ml/" + this.getClass.getSimpleName
|
||||
}
|
||||
|
||||
object PostNuxMlParams {
|
||||
|
||||
// infra params:
|
||||
case object FetchCandidateSourceBudget extends PostNuxMlParams[Duration](90.millisecond)
|
||||
|
||||
// WTF Impression Store has very high tail latency (p9990 or p9999), but p99 latency is pretty good (~100ms)
|
||||
// set the time budget for this step to be 200ms to make the performance of service more predictable
|
||||
case object FatigueRankerBudget extends PostNuxMlParams[Duration](200.millisecond)
|
||||
|
||||
case object MlRankerBudget
|
||||
extends FSBoundedParam[Duration](
|
||||
name = PostNuxMlFlowFeatureSwitchKeys.MLRankerBudget,
|
||||
default = 400.millisecond,
|
||||
min = 100.millisecond,
|
||||
max = 800.millisecond)
|
||||
with HasDurationConversion {
|
||||
override val durationConversion: DurationConversion = DurationConversion.FromMillis
|
||||
}
|
||||
|
||||
// product params:
|
||||
case object TargetEligibility extends PostNuxMlParams[Boolean](true)
|
||||
|
||||
case object ResultSizeParam extends PostNuxMlParams[Int](3)
|
||||
case object BatchSizeParam extends PostNuxMlParams[Int](12)
|
||||
|
||||
case object CandidateShuffler
|
||||
extends PostNuxMlParams[CandidateShuffler[CandidateUser]](
|
||||
new ExponentialShuffler[CandidateUser])
|
||||
case object LogRandomRankerId extends PostNuxMlParams[Boolean](false)
|
||||
|
||||
// whether or not to use the ml ranker at all (feature hydration + ranker)
|
||||
case object UseMlRanker
|
||||
extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.UseMlRanker, false)
|
||||
|
||||
// whether or not to enable candidate param hydration in postnux_ml_flow
|
||||
case object EnableCandidateParamHydration
|
||||
extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableCandidateParamHydration, false)
|
||||
|
||||
// Whether or not OnlineSTP candidates are considered in the final pool of candidates.
|
||||
// If set to `false`, the candidate source will be removed *after* all other considerations.
|
||||
case object OnlineSTPEnabled
|
||||
extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.OnlineSTPEnabled, false)
|
||||
|
||||
// Whether or not the candidates are sampled from a Plackett-Luce model
|
||||
case object SamplingTransformEnabled
|
||||
extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.SamplingTransformEnabled, false)
|
||||
|
||||
// Whether or not Follow2Vec candidates are considered in the final pool of candidates.
|
||||
// If set to `false`, the candidate source will be removed *after* all other considerations.
|
||||
case object Follow2VecLinearRegressionEnabled
|
||||
extends FSParam[Boolean](
|
||||
PostNuxMlFlowFeatureSwitchKeys.Follow2VecLinearRegressionEnabled,
|
||||
false)
|
||||
|
||||
// Whether or not to enable AdhocRanker to allow adhoc, non-ML, score modifications.
|
||||
case object EnableAdhocRanker
|
||||
extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableAdhocRanker, false)
|
||||
|
||||
// Whether the impression-based fatigue ranker is enabled or not.
|
||||
case object EnableFatigueRanker
|
||||
extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableFatigueRanker, true)
|
||||
|
||||
// whether or not to enable InterleaveRanker for producer-side experiments.
|
||||
case object EnableInterleaveRanker
|
||||
extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableInterleaveRanker, false)
|
||||
|
||||
// whether to exclude users in near zero user state
|
||||
case object ExcludeNearZeroCandidates
|
||||
extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.ExcludeNearZeroCandidates, false)
|
||||
|
||||
case object EnablePPMILocaleFollowSourceInPostNux
|
||||
extends FSParam[Boolean](
|
||||
PostNuxMlFlowFeatureSwitchKeys.EnablePPMILocaleFollowSourceInPostNux,
|
||||
false)
|
||||
|
||||
case object EnableInterestsOptOutPredicate
|
||||
extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableInterestsOptOutPredicate, false)
|
||||
|
||||
case object EnableInvalidRelationshipPredicate
|
||||
extends FSParam[Boolean](
|
||||
PostNuxMlFlowFeatureSwitchKeys.EnableInvalidRelationshipPredicate,
|
||||
false)
|
||||
|
||||
// Totally disabling SGS predicate need to disable EnableInvalidRelationshipPredicate as well
|
||||
case object EnableSGSPredicate
|
||||
extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableSGSPredicate, true)
|
||||
|
||||
case object EnableHssPredicate
|
||||
extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableHssPredicate, true)
|
||||
|
||||
// Whether or not to include RepeatedProfileVisits as one of the candidate sources in the PostNuxMlFlow. If false,
|
||||
// RepeatedProfileVisitsSource would not be run for the users in candidate_generation.
|
||||
case object IncludeRepeatedProfileVisitsCandidateSource
|
||||
extends FSParam[Boolean](
|
||||
PostNuxMlFlowFeatureSwitchKeys.IncludeRepeatedProfileVisitsCandidateSource,
|
||||
false)
|
||||
|
||||
case object EnableRealGraphOonV2
|
||||
extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.EnableRealGraphOonV2, false)
|
||||
|
||||
case object GetFollowersFromSgs
|
||||
extends FSParam[Boolean](PostNuxMlFlowFeatureSwitchKeys.GetFollowersFromSgs, false)
|
||||
|
||||
case object EnableRemoveAccountProofTransform
|
||||
extends FSParam[Boolean](
|
||||
PostNuxMlFlowFeatureSwitchKeys.EnableRemoveAccountProofTransform,
|
||||
false)
|
||||
|
||||
// quality factor threshold to turn off ML ranker completely
|
||||
object TurnoffMLScorerQFThreshold
|
||||
extends FSBoundedParam[Double](
|
||||
name = PostNuxMlFlowFeatureSwitchKeys.TurnOffMLScorerQFThreshold,
|
||||
default = 0.3,
|
||||
min = 0.1,
|
||||
max = 1.0)
|
||||
}
|
||||
@ -0,0 +1,54 @@
|
||||
package com.twitter.follow_recommendations.flows.post_nux_ml
|
||||
|
||||
import com.twitter.core_workflows.user_model.thriftscala.UserState
|
||||
import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature
|
||||
import com.twitter.follow_recommendations.common.models._
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.ClientContext
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext
|
||||
import com.twitter.timelines.configapi.HasParams
|
||||
import com.twitter.timelines.configapi.Params
|
||||
|
||||
case class PostNuxMlRequest(
|
||||
override val params: Params,
|
||||
override val clientContext: ClientContext,
|
||||
override val similarToUserIds: Seq[Long],
|
||||
inputExcludeUserIds: Seq[Long],
|
||||
override val recentFollowedUserIds: Option[Seq[Long]],
|
||||
override val invalidRelationshipUserIds: Option[Set[Long]],
|
||||
override val recentFollowedByUserIds: Option[Seq[Long]],
|
||||
override val dismissedUserIds: Option[Seq[Long]],
|
||||
override val displayLocation: DisplayLocation,
|
||||
maxResults: Option[Int] = None,
|
||||
override val debugOptions: Option[DebugOptions] = None,
|
||||
override val wtfImpressions: Option[Seq[WtfImpression]],
|
||||
override val uttInterestIds: Option[Seq[Long]] = None,
|
||||
override val customInterests: Option[Seq[String]] = None,
|
||||
override val geohashAndCountryCode: Option[GeohashAndCountryCode] = None,
|
||||
inputPreviouslyRecommendedUserIds: Option[Set[Long]] = None,
|
||||
inputPreviouslyFollowedUserIds: Option[Set[Long]] = None,
|
||||
override val isSoftUser: Boolean = false,
|
||||
override val userState: Option[UserState] = None,
|
||||
override val qualityFactor: Option[Double] = None)
|
||||
extends HasParams
|
||||
with HasSimilarToContext
|
||||
with HasClientContext
|
||||
with HasExcludedUserIds
|
||||
with HasDisplayLocation
|
||||
with HasDebugOptions
|
||||
with HasGeohashAndCountryCode
|
||||
with HasPreFetchedFeature
|
||||
with HasDismissedUserIds
|
||||
with HasInterestIds
|
||||
with HasPreviousRecommendationsContext
|
||||
with HasIsSoftUser
|
||||
with HasUserState
|
||||
with HasInvalidRelationshipUserIds
|
||||
with HasQualityFactor {
|
||||
override val excludedUserIds: Seq[Long] = {
|
||||
inputExcludeUserIds ++ clientContext.userId.toSeq ++ similarToUserIds
|
||||
}
|
||||
override val previouslyRecommendedUserIDs: Set[Long] =
|
||||
inputPreviouslyRecommendedUserIds.getOrElse(Set.empty)
|
||||
override val previouslyFollowedUserIds: Set[Long] =
|
||||
inputPreviouslyFollowedUserIds.getOrElse(Set.empty)
|
||||
}
|
||||
@ -0,0 +1,173 @@
|
||||
package com.twitter.follow_recommendations.flows.post_nux_ml
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.clients.dismiss_store.DismissStore
|
||||
import com.twitter.follow_recommendations.common.clients.geoduck.UserLocationFetcher
|
||||
import com.twitter.follow_recommendations.common.clients.impression_store.WtfImpressionStore
|
||||
import com.twitter.follow_recommendations.common.clients.interests_service.InterestServiceClient
|
||||
import com.twitter.follow_recommendations.common.clients.socialgraph.SocialGraphClient
|
||||
import com.twitter.follow_recommendations.common.clients.user_state.UserStateClient
|
||||
import com.twitter.follow_recommendations.common.predicates.dismiss.DismissedCandidatePredicateParams
|
||||
import com.twitter.follow_recommendations.common.utils.RescueWithStatsUtils._
|
||||
import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlRequestBuilderParams.DismissedIdScanBudget
|
||||
import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlRequestBuilderParams.TopicIdFetchBudget
|
||||
import com.twitter.follow_recommendations.flows.post_nux_ml.PostNuxMlRequestBuilderParams.WTFImpressionsScanBudget
|
||||
import com.twitter.follow_recommendations.products.common.ProductRequest
|
||||
import com.twitter.inject.Logging
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.util.Time
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Singleton
|
||||
|
||||
@Singleton
|
||||
class PostNuxMlRequestBuilder @Inject() (
|
||||
socialGraph: SocialGraphClient,
|
||||
wtfImpressionStore: WtfImpressionStore,
|
||||
dismissStore: DismissStore,
|
||||
userLocationFetcher: UserLocationFetcher,
|
||||
interestServiceClient: InterestServiceClient,
|
||||
userStateClient: UserStateClient,
|
||||
statsReceiver: StatsReceiver)
|
||||
extends Logging {
|
||||
|
||||
val stats: StatsReceiver = statsReceiver.scope("post_nux_ml_request_builder")
|
||||
val invalidRelationshipUsersStats: StatsReceiver = stats.scope("invalidRelationshipUserIds")
|
||||
private val invalidRelationshipUsersMaxSizeCounter =
|
||||
invalidRelationshipUsersStats.counter("maxSize")
|
||||
private val invalidRelationshipUsersNotMaxSizeCounter =
|
||||
invalidRelationshipUsersStats.counter("notMaxSize")
|
||||
|
||||
def build(
|
||||
req: ProductRequest,
|
||||
previouslyRecommendedUserIds: Option[Set[Long]] = None,
|
||||
previouslyFollowedUserIds: Option[Set[Long]] = None
|
||||
): Stitch[PostNuxMlRequest] = {
|
||||
val dl = req.recommendationRequest.displayLocation
|
||||
val resultsStitch = Stitch.collect(
|
||||
req.recommendationRequest.clientContext.userId
|
||||
.map { userId =>
|
||||
val lookBackDuration = req.params(DismissedCandidatePredicateParams.LookBackDuration)
|
||||
val negativeStartTs = -(Time.now - lookBackDuration).inMillis
|
||||
val recentFollowedUserIdsStitch =
|
||||
rescueWithStats(
|
||||
socialGraph.getRecentFollowedUserIds(userId),
|
||||
stats,
|
||||
"recentFollowedUserIds")
|
||||
val invalidRelationshipUserIdsStitch =
|
||||
if (req.params(PostNuxMlParams.EnableInvalidRelationshipPredicate)) {
|
||||
rescueWithStats(
|
||||
socialGraph
|
||||
.getInvalidRelationshipUserIds(userId)
|
||||
.onSuccess(ids =>
|
||||
if (ids.size >= SocialGraphClient.MaxNumInvalidRelationship) {
|
||||
invalidRelationshipUsersMaxSizeCounter.incr()
|
||||
} else {
|
||||
invalidRelationshipUsersNotMaxSizeCounter.incr()
|
||||
}),
|
||||
stats,
|
||||
"invalidRelationshipUserIds"
|
||||
)
|
||||
} else {
|
||||
Stitch.value(Seq.empty)
|
||||
}
|
||||
// recentFollowedByUserIds are only used in experiment candidate sources
|
||||
val recentFollowedByUserIdsStitch = if (req.params(PostNuxMlParams.GetFollowersFromSgs)) {
|
||||
rescueWithStats(
|
||||
socialGraph.getRecentFollowedByUserIdsFromCachedColumn(userId),
|
||||
stats,
|
||||
"recentFollowedByUserIds")
|
||||
} else Stitch.value(Seq.empty)
|
||||
val wtfImpressionsStitch =
|
||||
rescueWithStatsWithin(
|
||||
wtfImpressionStore.get(userId, dl),
|
||||
stats,
|
||||
"wtfImpressions",
|
||||
req.params(WTFImpressionsScanBudget))
|
||||
val dismissedUserIdsStitch =
|
||||
rescueWithStatsWithin(
|
||||
dismissStore.get(userId, negativeStartTs, None),
|
||||
stats,
|
||||
"dismissedUserIds",
|
||||
req.params(DismissedIdScanBudget))
|
||||
val locationStitch =
|
||||
rescueOptionalWithStats(
|
||||
userLocationFetcher.getGeohashAndCountryCode(
|
||||
Some(userId),
|
||||
req.recommendationRequest.clientContext.ipAddress),
|
||||
stats,
|
||||
"userLocation"
|
||||
)
|
||||
val topicIdsStitch =
|
||||
rescueWithStatsWithin(
|
||||
interestServiceClient.fetchUttInterestIds(userId),
|
||||
stats,
|
||||
"topicIds",
|
||||
req.params(TopicIdFetchBudget))
|
||||
val userStateStitch =
|
||||
rescueOptionalWithStats(userStateClient.getUserState(userId), stats, "userState")
|
||||
Stitch.join(
|
||||
recentFollowedUserIdsStitch,
|
||||
invalidRelationshipUserIdsStitch,
|
||||
recentFollowedByUserIdsStitch,
|
||||
dismissedUserIdsStitch,
|
||||
wtfImpressionsStitch,
|
||||
locationStitch,
|
||||
topicIdsStitch,
|
||||
userStateStitch
|
||||
)
|
||||
})
|
||||
|
||||
resultsStitch.map {
|
||||
case Some(
|
||||
(
|
||||
recentFollowedUserIds,
|
||||
invalidRelationshipUserIds,
|
||||
recentFollowedByUserIds,
|
||||
dismissedUserIds,
|
||||
wtfImpressions,
|
||||
locationInfo,
|
||||
topicIds,
|
||||
userState)) =>
|
||||
PostNuxMlRequest(
|
||||
params = req.params,
|
||||
clientContext = req.recommendationRequest.clientContext,
|
||||
similarToUserIds = Nil,
|
||||
inputExcludeUserIds = req.recommendationRequest.excludedIds.getOrElse(Nil),
|
||||
recentFollowedUserIds = Some(recentFollowedUserIds),
|
||||
invalidRelationshipUserIds = Some(invalidRelationshipUserIds.toSet),
|
||||
recentFollowedByUserIds = Some(recentFollowedByUserIds),
|
||||
dismissedUserIds = Some(dismissedUserIds),
|
||||
displayLocation = dl,
|
||||
maxResults = req.recommendationRequest.maxResults,
|
||||
debugOptions = req.recommendationRequest.debugParams.flatMap(_.debugOptions),
|
||||
wtfImpressions = Some(wtfImpressions),
|
||||
geohashAndCountryCode = locationInfo,
|
||||
uttInterestIds = Some(topicIds),
|
||||
inputPreviouslyRecommendedUserIds = previouslyRecommendedUserIds,
|
||||
inputPreviouslyFollowedUserIds = previouslyFollowedUserIds,
|
||||
isSoftUser = req.recommendationRequest.isSoftUser,
|
||||
userState = userState
|
||||
)
|
||||
case _ =>
|
||||
PostNuxMlRequest(
|
||||
params = req.params,
|
||||
clientContext = req.recommendationRequest.clientContext,
|
||||
similarToUserIds = Nil,
|
||||
inputExcludeUserIds = req.recommendationRequest.excludedIds.getOrElse(Nil),
|
||||
recentFollowedUserIds = None,
|
||||
invalidRelationshipUserIds = None,
|
||||
recentFollowedByUserIds = None,
|
||||
dismissedUserIds = None,
|
||||
displayLocation = dl,
|
||||
maxResults = req.recommendationRequest.maxResults,
|
||||
debugOptions = req.recommendationRequest.debugParams.flatMap(_.debugOptions),
|
||||
wtfImpressions = None,
|
||||
geohashAndCountryCode = None,
|
||||
inputPreviouslyRecommendedUserIds = previouslyRecommendedUserIds,
|
||||
inputPreviouslyFollowedUserIds = previouslyFollowedUserIds,
|
||||
isSoftUser = req.recommendationRequest.isSoftUser,
|
||||
userState = None
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,45 @@
|
||||
package com.twitter.follow_recommendations.flows.post_nux_ml
|
||||
|
||||
import com.twitter.timelines.configapi.FSBoundedParam
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.timelines.configapi.DurationConversion
|
||||
import com.twitter.timelines.configapi.FSParam
|
||||
import com.twitter.timelines.configapi.HasDurationConversion
|
||||
|
||||
object PostNuxMlRequestBuilderParams {
|
||||
case object TopicIdFetchBudget
|
||||
extends FSBoundedParam[Duration](
|
||||
name = "post_nux_ml_request_builder_topic_id_fetch_budget_millis",
|
||||
default = 200.millisecond,
|
||||
min = 80.millisecond,
|
||||
max = 400.millisecond)
|
||||
with HasDurationConversion {
|
||||
override val durationConversion: DurationConversion = DurationConversion.FromMillis
|
||||
}
|
||||
|
||||
case object DismissedIdScanBudget
|
||||
extends FSBoundedParam[Duration](
|
||||
name = "post_nux_ml_request_builder_dismissed_id_scan_budget_millis",
|
||||
default = 200.millisecond,
|
||||
min = 80.millisecond,
|
||||
max = 400.millisecond)
|
||||
with HasDurationConversion {
|
||||
override val durationConversion: DurationConversion = DurationConversion.FromMillis
|
||||
}
|
||||
|
||||
case object WTFImpressionsScanBudget
|
||||
extends FSBoundedParam[Duration](
|
||||
name = "post_nux_ml_request_builder_wtf_impressions_scan_budget_millis",
|
||||
default = 200.millisecond,
|
||||
min = 80.millisecond,
|
||||
max = 400.millisecond)
|
||||
with HasDurationConversion {
|
||||
override val durationConversion: DurationConversion = DurationConversion.FromMillis
|
||||
}
|
||||
|
||||
case object EnableInvalidRelationshipPredicate
|
||||
extends FSParam[Boolean](
|
||||
name = "post_nux_ml_request_builder_enable_invalid_relationship_predicate",
|
||||
false)
|
||||
}
|
||||
@ -0,0 +1,18 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/com/google/inject/extensions:guice-assistedinject",
|
||||
"3rdparty/jvm/net/codingwell:scala-guice",
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi/params",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/models",
|
||||
"follow-recommendations-service/thrift/src/main/thrift:thrift-scala",
|
||||
"scribelib/marshallers/src/main/scala/com/twitter/scribelib/marshallers",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,164 @@
|
||||
package com.twitter.follow_recommendations.logging
|
||||
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants
|
||||
import com.twitter.follow_recommendations.common.models.HasIsSoftUser
|
||||
import com.twitter.follow_recommendations.configapi.params.GlobalParams
|
||||
import com.twitter.follow_recommendations.logging.thriftscala.RecommendationLog
|
||||
import com.twitter.follow_recommendations.models.DebugParams
|
||||
import com.twitter.follow_recommendations.models.RecommendationFlowData
|
||||
import com.twitter.follow_recommendations.models.RecommendationRequest
|
||||
import com.twitter.follow_recommendations.models.RecommendationResponse
|
||||
import com.twitter.follow_recommendations.models.ScoringUserRequest
|
||||
import com.twitter.follow_recommendations.models.ScoringUserResponse
|
||||
import com.twitter.inject.annotations.Flag
|
||||
import com.twitter.logging.LoggerFactory
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.ClientContext
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext
|
||||
import com.twitter.scribelib.marshallers.ClientDataProvider
|
||||
import com.twitter.scribelib.marshallers.ExternalRefererDataProvider
|
||||
import com.twitter.scribelib.marshallers.ScribeSerialization
|
||||
import com.twitter.timelines.configapi.HasParams
|
||||
import com.twitter.util.Time
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Named
|
||||
import javax.inject.Singleton
|
||||
|
||||
/**
|
||||
* This is the standard logging class we use to log data into:
|
||||
* 1) logs.follow_recommendations_logs
|
||||
*
|
||||
* This logger logs data for 2 endpoints: getRecommendations, scoreUserCandidates
|
||||
* All data scribed via this logger have to be converted into the same thrift type: RecommendationLog
|
||||
*
|
||||
* 2) logs.frs_recommendation_flow_logs
|
||||
*
|
||||
* This logger logs recommendation flow data for getRecommendations requests
|
||||
* All data scribed via this logger have to be converted into the same thrift type: FrsRecommendationFlowLog
|
||||
*/
|
||||
@Singleton
|
||||
class FrsLogger @Inject() (
|
||||
@Named(GuiceNamedConstants.REQUEST_LOGGER) loggerFactory: LoggerFactory,
|
||||
@Named(GuiceNamedConstants.FLOW_LOGGER) flowLoggerFactory: LoggerFactory,
|
||||
stats: StatsReceiver,
|
||||
@Flag("log_results") serviceShouldLogResults: Boolean)
|
||||
extends ScribeSerialization {
|
||||
private val logger = loggerFactory.apply()
|
||||
private val flowLogger = flowLoggerFactory.apply()
|
||||
private val logRecommendationCounter = stats.counter("scribe_recommendation")
|
||||
private val logScoringCounter = stats.counter("scribe_scoring")
|
||||
private val logRecommendationFlowCounter = stats.counter("scribe_recommendation_flow")
|
||||
|
||||
def logRecommendationResult(
|
||||
request: RecommendationRequest,
|
||||
response: RecommendationResponse
|
||||
): Unit = {
|
||||
if (!request.isSoftUser) {
|
||||
val log =
|
||||
RecommendationLog(request.toOfflineThrift, response.toOfflineThrift, Time.now.inMillis)
|
||||
logRecommendationCounter.incr()
|
||||
logger.info(
|
||||
serializeThrift(
|
||||
log,
|
||||
FrsLogger.LogCategory,
|
||||
FrsLogger.mkProvider(request.clientContext)
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
def logScoringResult(request: ScoringUserRequest, response: ScoringUserResponse): Unit = {
|
||||
if (!request.isSoftUser) {
|
||||
val log =
|
||||
RecommendationLog(
|
||||
request.toRecommendationRequest.toOfflineThrift,
|
||||
response.toRecommendationResponse.toOfflineThrift,
|
||||
Time.now.inMillis)
|
||||
logScoringCounter.incr()
|
||||
logger.info(
|
||||
serializeThrift(
|
||||
log,
|
||||
FrsLogger.LogCategory,
|
||||
FrsLogger.mkProvider(request.toRecommendationRequest.clientContext)
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
def logRecommendationFlowData[Target <: HasClientContext with HasIsSoftUser with HasParams](
|
||||
request: Target,
|
||||
flowData: RecommendationFlowData[Target]
|
||||
): Unit = {
|
||||
if (!request.isSoftUser && request.params(GlobalParams.EnableRecommendationFlowLogs)) {
|
||||
val log = flowData.toRecommendationFlowLogOfflineThrift
|
||||
logRecommendationFlowCounter.incr()
|
||||
flowLogger.info(
|
||||
serializeThrift(
|
||||
log,
|
||||
FrsLogger.FlowLogCategory,
|
||||
FrsLogger.mkProvider(request.clientContext)
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
// We prefer the settings given in the user request, and if none provided we default to the
|
||||
// aurora service configuration.
|
||||
def shouldLog(debugParamsOpt: Option[DebugParams]): Boolean =
|
||||
debugParamsOpt match {
|
||||
case Some(debugParams) =>
|
||||
debugParams.debugOptions match {
|
||||
case Some(debugOptions) =>
|
||||
!debugOptions.doNotLog
|
||||
case None =>
|
||||
serviceShouldLogResults
|
||||
}
|
||||
case None =>
|
||||
serviceShouldLogResults
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
object FrsLogger {
|
||||
val LogCategory = "follow_recommendations_logs"
|
||||
val FlowLogCategory = "frs_recommendation_flow_logs"
|
||||
|
||||
def mkProvider(clientContext: ClientContext) = new ClientDataProvider {
|
||||
|
||||
/** The id of the current user. When the user is logged out, this method should return None. */
|
||||
override val userId: Option[Long] = clientContext.userId
|
||||
|
||||
/** The id of the guest, which is present in logged-in or loged-out states */
|
||||
override val guestId: Option[Long] = clientContext.guestId
|
||||
|
||||
/** The personalization id (pid) of the user, used to personalize Twitter services */
|
||||
override val personalizationId: Option[String] = None
|
||||
|
||||
/** The id of the individual device the user is currently using. This id will be unique for different users' devices. */
|
||||
override val deviceId: Option[String] = clientContext.deviceId
|
||||
|
||||
/** The OAuth application id of the application the user is currently using */
|
||||
override val clientApplicationId: Option[Long] = clientContext.appId
|
||||
|
||||
/** The OAuth parent application id of the application the user is currently using */
|
||||
override val parentApplicationId: Option[Long] = None
|
||||
|
||||
/** The two-letter, upper-case country code used to designate the country from which the scribe event occurred */
|
||||
override val countryCode: Option[String] = clientContext.countryCode
|
||||
|
||||
/** The two-letter, lower-case language code used to designate the probably language spoken by the scribe event initiator */
|
||||
override val languageCode: Option[String] = clientContext.languageCode
|
||||
|
||||
/** The user-agent header used to identify the client browser or device that the user is currently active on */
|
||||
override val userAgent: Option[String] = clientContext.userAgent
|
||||
|
||||
/** Whether the user is accessing Twitter via a secured connection */
|
||||
override val isSsl: Option[Boolean] = Some(true)
|
||||
|
||||
/** The referring URL to the current page for web-based clients, if applicable */
|
||||
override val referer: Option[String] = None
|
||||
|
||||
/**
|
||||
* The external site, partner, or email that lead to the current Twitter application. Returned value consists of a
|
||||
* tuple including the encrypted referral data and the type of referral
|
||||
*/
|
||||
override val externalReferer: Option[ExternalRefererDataProvider] = None
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,13 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"configapi/configapi-core/src/main/scala/com/twitter/timelines/configapi",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/feature_hydration/common",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/models",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/assembler/models",
|
||||
"follow-recommendations-service/thrift/src/main/thrift:thrift-scala",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/model/marshalling/request",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,9 @@
|
||||
package com.twitter.follow_recommendations.models
|
||||
|
||||
object CandidateSourceType extends Enumeration {
|
||||
type CandidateSourceType = Value
|
||||
val Social = Value("social")
|
||||
val GeoAndInterests = Value("geo_and_interests")
|
||||
val ActivityContextual = Value("activity_contextual")
|
||||
val None = Value("none")
|
||||
}
|
||||
@ -0,0 +1,5 @@
|
||||
package com.twitter.follow_recommendations.models
|
||||
|
||||
import com.twitter.timelines.configapi.Params
|
||||
|
||||
case class CandidateUserDebugParams(paramsMap: Map[Long, Params])
|
||||
@ -0,0 +1,28 @@
|
||||
package com.twitter.follow_recommendations.models
|
||||
|
||||
import com.twitter.follow_recommendations.common.models.DebugOptions
|
||||
import com.twitter.follow_recommendations.common.models.DebugOptions.fromDebugParamsThrift
|
||||
import com.twitter.follow_recommendations.logging.{thriftscala => offline}
|
||||
import com.twitter.follow_recommendations.{thriftscala => t}
|
||||
import com.twitter.timelines.configapi.{FeatureValue => ConfigApiFeatureValue}
|
||||
|
||||
case class DebugParams(
|
||||
featureOverrides: Option[Map[String, ConfigApiFeatureValue]],
|
||||
debugOptions: Option[DebugOptions])
|
||||
|
||||
object DebugParams {
|
||||
def fromThrift(thrift: t.DebugParams): DebugParams = DebugParams(
|
||||
featureOverrides = thrift.featureOverrides.map { map =>
|
||||
map.mapValues(FeatureValue.fromThrift).toMap
|
||||
},
|
||||
debugOptions = Some(
|
||||
fromDebugParamsThrift(thrift)
|
||||
)
|
||||
)
|
||||
def toOfflineThrift(model: DebugParams): offline.OfflineDebugParams =
|
||||
offline.OfflineDebugParams(randomizationSeed = model.debugOptions.flatMap(_.randomizationSeed))
|
||||
}
|
||||
|
||||
trait HasFrsDebugParams {
|
||||
def frsDebugParams: Option[DebugParams]
|
||||
}
|
||||
@ -0,0 +1,113 @@
|
||||
package com.twitter.follow_recommendations.models
|
||||
|
||||
import com.twitter.follow_recommendations.common.models.FlowContext
|
||||
import com.twitter.follow_recommendations.common.models.RecentlyEngagedUserId
|
||||
import com.twitter.follow_recommendations.logging.thriftscala.OfflineDisplayContext
|
||||
import com.twitter.follow_recommendations.logging.{thriftscala => offline}
|
||||
import com.twitter.follow_recommendations.{thriftscala => t}
|
||||
import scala.reflect.ClassTag
|
||||
import scala.reflect.classTag
|
||||
|
||||
trait DisplayContext {
|
||||
def toOfflineThrift: offline.OfflineDisplayContext
|
||||
}
|
||||
|
||||
object DisplayContext {
|
||||
case class Profile(profileId: Long) extends DisplayContext {
|
||||
override val toOfflineThrift: OfflineDisplayContext =
|
||||
offline.OfflineDisplayContext.Profile(offline.OfflineProfile(profileId))
|
||||
}
|
||||
case class Search(searchQuery: String) extends DisplayContext {
|
||||
override val toOfflineThrift: OfflineDisplayContext =
|
||||
offline.OfflineDisplayContext.Search(offline.OfflineSearch(searchQuery))
|
||||
}
|
||||
case class Rux(focalAuthorId: Long) extends DisplayContext {
|
||||
override val toOfflineThrift: OfflineDisplayContext =
|
||||
offline.OfflineDisplayContext.Rux(offline.OfflineRux(focalAuthorId))
|
||||
}
|
||||
|
||||
case class Topic(topicId: Long) extends DisplayContext {
|
||||
override val toOfflineThrift: OfflineDisplayContext =
|
||||
offline.OfflineDisplayContext.Topic(offline.OfflineTopic(topicId))
|
||||
}
|
||||
|
||||
case class ReactiveFollow(followedUserIds: Seq[Long]) extends DisplayContext {
|
||||
override val toOfflineThrift: OfflineDisplayContext =
|
||||
offline.OfflineDisplayContext.ReactiveFollow(offline.OfflineReactiveFollow(followedUserIds))
|
||||
}
|
||||
|
||||
case class NuxInterests(flowContext: Option[FlowContext], uttInterestIds: Option[Seq[Long]])
|
||||
extends DisplayContext {
|
||||
override val toOfflineThrift: OfflineDisplayContext =
|
||||
offline.OfflineDisplayContext.NuxInterests(
|
||||
offline.OfflineNuxInterests(flowContext.map(_.toOfflineThrift)))
|
||||
}
|
||||
|
||||
case class PostNuxFollowTask(flowContext: Option[FlowContext]) extends DisplayContext {
|
||||
override val toOfflineThrift: OfflineDisplayContext =
|
||||
offline.OfflineDisplayContext.PostNuxFollowTask(
|
||||
offline.OfflinePostNuxFollowTask(flowContext.map(_.toOfflineThrift)))
|
||||
}
|
||||
|
||||
case class AdCampaignTarget(similarToUserIds: Seq[Long]) extends DisplayContext {
|
||||
override val toOfflineThrift: OfflineDisplayContext =
|
||||
offline.OfflineDisplayContext.AdCampaignTarget(
|
||||
offline.OfflineAdCampaignTarget(similarToUserIds))
|
||||
}
|
||||
|
||||
case class ConnectTab(
|
||||
byfSeedUserIds: Seq[Long],
|
||||
similarToUserIds: Seq[Long],
|
||||
engagedUserIds: Seq[RecentlyEngagedUserId])
|
||||
extends DisplayContext {
|
||||
override val toOfflineThrift: OfflineDisplayContext =
|
||||
offline.OfflineDisplayContext.ConnectTab(
|
||||
offline.OfflineConnectTab(
|
||||
byfSeedUserIds,
|
||||
similarToUserIds,
|
||||
engagedUserIds.map(user => user.toOfflineThrift)))
|
||||
}
|
||||
|
||||
case class SimilarToUser(similarToUserId: Long) extends DisplayContext {
|
||||
override val toOfflineThrift: OfflineDisplayContext =
|
||||
offline.OfflineDisplayContext.SimilarToUser(offline.OfflineSimilarToUser(similarToUserId))
|
||||
}
|
||||
|
||||
def fromThrift(tDisplayContext: t.DisplayContext): DisplayContext = tDisplayContext match {
|
||||
case t.DisplayContext.Profile(p) => Profile(p.profileId)
|
||||
case t.DisplayContext.Search(s) => Search(s.searchQuery)
|
||||
case t.DisplayContext.Rux(r) => Rux(r.focalAuthorId)
|
||||
case t.DisplayContext.Topic(t) => Topic(t.topicId)
|
||||
case t.DisplayContext.ReactiveFollow(f) => ReactiveFollow(f.followedUserIds)
|
||||
case t.DisplayContext.NuxInterests(n) =>
|
||||
NuxInterests(n.flowContext.map(FlowContext.fromThrift), n.uttInterestIds)
|
||||
case t.DisplayContext.AdCampaignTarget(a) =>
|
||||
AdCampaignTarget(a.similarToUserIds)
|
||||
case t.DisplayContext.ConnectTab(connect) =>
|
||||
ConnectTab(
|
||||
connect.byfSeedUserIds,
|
||||
connect.similarToUserIds,
|
||||
connect.recentlyEngagedUserIds.map(RecentlyEngagedUserId.fromThrift))
|
||||
case t.DisplayContext.SimilarToUser(r) =>
|
||||
SimilarToUser(r.similarToUserId)
|
||||
case t.DisplayContext.PostNuxFollowTask(p) =>
|
||||
PostNuxFollowTask(p.flowContext.map(FlowContext.fromThrift))
|
||||
case t.DisplayContext.UnknownUnionField(t) =>
|
||||
throw new UnknownDisplayContextException(t.field.name)
|
||||
}
|
||||
|
||||
def getDisplayContextAs[T <: DisplayContext: ClassTag](displayContext: DisplayContext): T =
|
||||
displayContext match {
|
||||
case context: T => context
|
||||
case _ =>
|
||||
throw new UnexpectedDisplayContextTypeException(
|
||||
displayContext,
|
||||
classTag[T].getClass.getSimpleName)
|
||||
}
|
||||
}
|
||||
|
||||
class UnknownDisplayContextException(name: String)
|
||||
extends Exception(s"Unknown DisplayContext in Thrift: ${name}")
|
||||
|
||||
class UnexpectedDisplayContextTypeException(displayContext: DisplayContext, expectedType: String)
|
||||
extends Exception(s"DisplayContext ${displayContext} not of expected type ${expectedType}")
|
||||
@ -0,0 +1,24 @@
|
||||
package com.twitter.follow_recommendations.models
|
||||
|
||||
import com.twitter.follow_recommendations.{thriftscala => t}
|
||||
import com.twitter.timelines.configapi._
|
||||
|
||||
object FeatureValue {
|
||||
def fromThrift(thriftFeatureValue: t.FeatureValue): FeatureValue = thriftFeatureValue match {
|
||||
case t.FeatureValue.PrimitiveValue(t.PrimitiveFeatureValue.BoolValue(bool)) =>
|
||||
BooleanFeatureValue(bool)
|
||||
case t.FeatureValue.PrimitiveValue(t.PrimitiveFeatureValue.StrValue(string)) =>
|
||||
StringFeatureValue(string)
|
||||
case t.FeatureValue.PrimitiveValue(t.PrimitiveFeatureValue.IntValue(int)) =>
|
||||
NumberFeatureValue(int)
|
||||
case t.FeatureValue.PrimitiveValue(t.PrimitiveFeatureValue.LongValue(long)) =>
|
||||
NumberFeatureValue(long)
|
||||
case t.FeatureValue.PrimitiveValue(t.PrimitiveFeatureValue.UnknownUnionField(field)) =>
|
||||
throw new UnknownFeatureValueException(s"Primitive: ${field.field.name}")
|
||||
case t.FeatureValue.UnknownUnionField(field) =>
|
||||
throw new UnknownFeatureValueException(field.field.name)
|
||||
}
|
||||
}
|
||||
|
||||
class UnknownFeatureValueException(fieldName: String)
|
||||
extends Exception(s"Unknown FeatureValue name in thrift: ${fieldName}")
|
||||
@ -0,0 +1,104 @@
|
||||
package com.twitter.follow_recommendations.models
|
||||
|
||||
import com.twitter.follow_recommendations.common.models.CandidateUser
|
||||
import com.twitter.follow_recommendations.common.models.ClientContextConverter
|
||||
import com.twitter.follow_recommendations.common.models.HasUserState
|
||||
import com.twitter.follow_recommendations.common.utils.UserSignupUtil
|
||||
import com.twitter.follow_recommendations.logging.{thriftscala => offline}
|
||||
import com.twitter.product_mixer.core.functional_component.candidate_source.CandidateSource
|
||||
import com.twitter.product_mixer.core.model.common.identifier.CandidateSourceIdentifier
|
||||
import com.twitter.product_mixer.core.model.common.identifier.RecommendationPipelineIdentifier
|
||||
import com.twitter.product_mixer.core.model.marshalling.HasMarshalling
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext
|
||||
import com.twitter.util.Time
|
||||
|
||||
case class RecommendationFlowData[Target <: HasClientContext](
|
||||
request: Target,
|
||||
recommendationFlowIdentifier: RecommendationPipelineIdentifier,
|
||||
candidateSources: Seq[CandidateSource[Target, CandidateUser]],
|
||||
candidatesFromCandidateSources: Seq[CandidateUser],
|
||||
mergedCandidates: Seq[CandidateUser],
|
||||
filteredCandidates: Seq[CandidateUser],
|
||||
rankedCandidates: Seq[CandidateUser],
|
||||
transformedCandidates: Seq[CandidateUser],
|
||||
truncatedCandidates: Seq[CandidateUser],
|
||||
results: Seq[CandidateUser])
|
||||
extends HasMarshalling {
|
||||
|
||||
import RecommendationFlowData._
|
||||
|
||||
lazy val toRecommendationFlowLogOfflineThrift: offline.RecommendationFlowLog = {
|
||||
val userMetadata = userToOfflineRecommendationFlowUserMetadata(request)
|
||||
val signals = userToOfflineRecommendationFlowSignals(request)
|
||||
val filteredCandidateSourceCandidates =
|
||||
candidatesToOfflineRecommendationFlowCandidateSourceCandidates(
|
||||
candidateSources,
|
||||
filteredCandidates
|
||||
)
|
||||
val rankedCandidateSourceCandidates =
|
||||
candidatesToOfflineRecommendationFlowCandidateSourceCandidates(
|
||||
candidateSources,
|
||||
rankedCandidates
|
||||
)
|
||||
val truncatedCandidateSourceCandidates =
|
||||
candidatesToOfflineRecommendationFlowCandidateSourceCandidates(
|
||||
candidateSources,
|
||||
truncatedCandidates
|
||||
)
|
||||
|
||||
offline.RecommendationFlowLog(
|
||||
ClientContextConverter.toFRSOfflineClientContextThrift(request.clientContext),
|
||||
userMetadata,
|
||||
signals,
|
||||
Time.now.inMillis,
|
||||
recommendationFlowIdentifier.name,
|
||||
Some(filteredCandidateSourceCandidates),
|
||||
Some(rankedCandidateSourceCandidates),
|
||||
Some(truncatedCandidateSourceCandidates)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
object RecommendationFlowData {
|
||||
def userToOfflineRecommendationFlowUserMetadata[Target <: HasClientContext](
|
||||
request: Target
|
||||
): Option[offline.OfflineRecommendationFlowUserMetadata] = {
|
||||
val userSignupAge = UserSignupUtil.userSignupAge(request).map(_.inDays)
|
||||
val userState = request match {
|
||||
case req: HasUserState => req.userState.map(_.name)
|
||||
case _ => None
|
||||
}
|
||||
Some(offline.OfflineRecommendationFlowUserMetadata(userSignupAge, userState))
|
||||
}
|
||||
|
||||
def userToOfflineRecommendationFlowSignals[Target <: HasClientContext](
|
||||
request: Target
|
||||
): Option[offline.OfflineRecommendationFlowSignals] = {
|
||||
val countryCode = request.getCountryCode
|
||||
Some(offline.OfflineRecommendationFlowSignals(countryCode))
|
||||
}
|
||||
|
||||
def candidatesToOfflineRecommendationFlowCandidateSourceCandidates[Target <: HasClientContext](
|
||||
candidateSources: Seq[CandidateSource[Target, CandidateUser]],
|
||||
candidates: Seq[CandidateUser],
|
||||
): Seq[offline.OfflineRecommendationFlowCandidateSourceCandidates] = {
|
||||
val candidatesGroupedByCandidateSources =
|
||||
candidates.groupBy(
|
||||
_.getPrimaryCandidateSource.getOrElse(CandidateSourceIdentifier("NoCandidateSource")))
|
||||
|
||||
candidateSources.map(candidateSource => {
|
||||
val candidates =
|
||||
candidatesGroupedByCandidateSources.get(candidateSource.identifier).toSeq.flatten
|
||||
val candidateUserIds = candidates.map(_.id)
|
||||
val candidateUserScores = candidates.map(_.score).exists(_.nonEmpty) match {
|
||||
case true => Some(candidates.map(_.score.getOrElse(-1.0)))
|
||||
case false => None
|
||||
}
|
||||
offline.OfflineRecommendationFlowCandidateSourceCandidates(
|
||||
candidateSource.identifier.name,
|
||||
candidateUserIds,
|
||||
candidateUserScores
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,29 @@
|
||||
package com.twitter.follow_recommendations.models
|
||||
|
||||
import com.twitter.follow_recommendations.common.models.ClientContextConverter
|
||||
import com.twitter.follow_recommendations.common.models.DisplayLocation
|
||||
import com.twitter.follow_recommendations.logging.{thriftscala => offline}
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.ClientContext
|
||||
|
||||
case class RecommendationRequest(
|
||||
clientContext: ClientContext,
|
||||
displayLocation: DisplayLocation,
|
||||
displayContext: Option[DisplayContext],
|
||||
maxResults: Option[Int],
|
||||
cursor: Option[String],
|
||||
excludedIds: Option[Seq[Long]],
|
||||
fetchPromotedContent: Option[Boolean],
|
||||
debugParams: Option[DebugParams] = None,
|
||||
userLocationState: Option[String] = None,
|
||||
isSoftUser: Boolean = false) {
|
||||
def toOfflineThrift: offline.OfflineRecommendationRequest = offline.OfflineRecommendationRequest(
|
||||
ClientContextConverter.toFRSOfflineClientContextThrift(clientContext),
|
||||
displayLocation.toOfflineThrift,
|
||||
displayContext.map(_.toOfflineThrift),
|
||||
maxResults,
|
||||
cursor,
|
||||
excludedIds,
|
||||
fetchPromotedContent,
|
||||
debugParams.map(DebugParams.toOfflineThrift)
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,14 @@
|
||||
package com.twitter.follow_recommendations.models
|
||||
|
||||
import com.twitter.follow_recommendations.{thriftscala => t}
|
||||
import com.twitter.follow_recommendations.logging.{thriftscala => offline}
|
||||
import com.twitter.follow_recommendations.common.models.Recommendation
|
||||
import com.twitter.product_mixer.core.model.marshalling.HasMarshalling
|
||||
|
||||
case class RecommendationResponse(recommendations: Seq[Recommendation]) extends HasMarshalling {
|
||||
lazy val toThrift: t.RecommendationResponse =
|
||||
t.RecommendationResponse(recommendations.map(_.toThrift))
|
||||
|
||||
lazy val toOfflineThrift: offline.OfflineRecommendationResponse =
|
||||
offline.OfflineRecommendationResponse(recommendations.map(_.toOfflineThrift))
|
||||
}
|
||||
@ -0,0 +1,22 @@
|
||||
package com.twitter.follow_recommendations.models
|
||||
|
||||
import com.twitter.follow_recommendations.common.models.DisplayLocation
|
||||
import com.twitter.product_mixer.core.model.marshalling.request
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.ClientContext
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.ProductContext
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.{Request => ProductMixerRequest}
|
||||
|
||||
case class Request(
|
||||
override val maxResults: Option[Int],
|
||||
override val debugParams: Option[request.DebugParams],
|
||||
override val productContext: Option[ProductContext],
|
||||
override val product: request.Product,
|
||||
override val clientContext: ClientContext,
|
||||
override val serializedRequestCursor: Option[String],
|
||||
override val frsDebugParams: Option[DebugParams],
|
||||
displayLocation: DisplayLocation,
|
||||
excludedIds: Option[Seq[Long]],
|
||||
fetchPromotedContent: Option[Boolean],
|
||||
userLocationState: Option[String] = None)
|
||||
extends ProductMixerRequest
|
||||
with HasFrsDebugParams {}
|
||||
@ -0,0 +1,45 @@
|
||||
package com.twitter.follow_recommendations.models
|
||||
|
||||
import com.twitter.follow_recommendations.common.feature_hydration.common.HasPreFetchedFeature
|
||||
import com.twitter.follow_recommendations.common.models._
|
||||
import com.twitter.follow_recommendations.logging.{thriftscala => offline}
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.ClientContext
|
||||
import com.twitter.product_mixer.core.model.marshalling.request.HasClientContext
|
||||
import com.twitter.timelines.configapi.HasParams
|
||||
import com.twitter.timelines.configapi.Params
|
||||
|
||||
case class ScoringUserRequest(
|
||||
override val clientContext: ClientContext,
|
||||
override val displayLocation: DisplayLocation,
|
||||
override val params: Params,
|
||||
override val debugOptions: Option[DebugOptions] = None,
|
||||
override val recentFollowedUserIds: Option[Seq[Long]],
|
||||
override val recentFollowedByUserIds: Option[Seq[Long]],
|
||||
override val wtfImpressions: Option[Seq[WtfImpression]],
|
||||
override val similarToUserIds: Seq[Long],
|
||||
candidates: Seq[CandidateUser],
|
||||
debugParams: Option[DebugParams] = None,
|
||||
isSoftUser: Boolean = false)
|
||||
extends HasClientContext
|
||||
with HasDisplayLocation
|
||||
with HasParams
|
||||
with HasDebugOptions
|
||||
with HasPreFetchedFeature
|
||||
with HasSimilarToContext {
|
||||
def toOfflineThrift: offline.OfflineScoringUserRequest = offline.OfflineScoringUserRequest(
|
||||
ClientContextConverter.toFRSOfflineClientContextThrift(clientContext),
|
||||
displayLocation.toOfflineThrift,
|
||||
candidates.map(_.toOfflineUserThrift)
|
||||
)
|
||||
def toRecommendationRequest: RecommendationRequest = RecommendationRequest(
|
||||
clientContext = clientContext,
|
||||
displayLocation = displayLocation,
|
||||
displayContext = None,
|
||||
maxResults = None,
|
||||
cursor = None,
|
||||
excludedIds = None,
|
||||
fetchPromotedContent = None,
|
||||
debugParams = debugParams,
|
||||
isSoftUser = isSoftUser
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,15 @@
|
||||
package com.twitter.follow_recommendations.models
|
||||
|
||||
import com.twitter.follow_recommendations.common.models.CandidateUser
|
||||
import com.twitter.follow_recommendations.logging.{thriftscala => offline}
|
||||
import com.twitter.follow_recommendations.{thriftscala => t}
|
||||
|
||||
case class ScoringUserResponse(candidates: Seq[CandidateUser]) {
|
||||
lazy val toThrift: t.ScoringUserResponse =
|
||||
t.ScoringUserResponse(candidates.map(_.toUserThrift))
|
||||
|
||||
lazy val toRecommendationResponse: RecommendationResponse = RecommendationResponse(candidates)
|
||||
|
||||
lazy val toOfflineThrift: offline.OfflineScoringUserResponse =
|
||||
offline.OfflineScoringUserResponse(candidates.map(_.toOfflineUserThrift))
|
||||
}
|
||||
@ -0,0 +1,8 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/pipeline/pipeline_failure",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,12 @@
|
||||
package com.twitter.follow_recommendations.models.failures
|
||||
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.CandidateSourceTimeout
|
||||
import com.twitter.product_mixer.core.pipeline.pipeline_failure.PipelineFailure
|
||||
|
||||
object TimeoutPipelineFailure {
|
||||
def apply(candidateSourceName: String): PipelineFailure = {
|
||||
PipelineFailure(
|
||||
CandidateSourceTimeout,
|
||||
s"Candidate Source $candidateSourceName timed out before returning candidates")
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,31 @@
|
||||
package com.twitter.follow_recommendations.modules
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.google.inject.name.Named
|
||||
import com.twitter.abdecider.ABDeciderFactory
|
||||
import com.twitter.abdecider.LoggingABDecider
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.logging.LoggerFactory
|
||||
import javax.inject.Singleton
|
||||
|
||||
object ABDeciderModule extends TwitterModule {
|
||||
@Provides
|
||||
@Singleton
|
||||
def provideABDecider(
|
||||
stats: StatsReceiver,
|
||||
@Named(GuiceNamedConstants.CLIENT_EVENT_LOGGER) factory: LoggerFactory
|
||||
): LoggingABDecider = {
|
||||
|
||||
val ymlPath = "/usr/local/config/abdecider/abdecider.yml"
|
||||
|
||||
val abDeciderFactory = ABDeciderFactory(
|
||||
abDeciderYmlPath = ymlPath,
|
||||
scribeLogger = Some(factory()),
|
||||
environment = Some("production")
|
||||
)
|
||||
|
||||
abDeciderFactory.buildWithLogging()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,24 @@
|
||||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"3rdparty/jvm/com/google/inject:guice",
|
||||
"3rdparty/jvm/com/google/inject/extensions:guice-assistedinject",
|
||||
"3rdparty/jvm/javax/inject:javax.inject",
|
||||
"3rdparty/jvm/net/codingwell:scala-guice",
|
||||
"3rdparty/jvm/org/slf4j:slf4j-api",
|
||||
"finatra-internal/mtls-thriftmux/src/main/scala",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/base",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/constants",
|
||||
"follow-recommendations-service/common/src/main/scala/com/twitter/follow_recommendations/common/transforms/modify_social_proof",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/configapi",
|
||||
"follow-recommendations-service/server/src/main/scala/com/twitter/follow_recommendations/products",
|
||||
"follow-recommendations-service/thrift/src/main/thrift:thrift-scala",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/guice",
|
||||
"product-mixer/core/src/main/scala/com/twitter/product_mixer/core/product/registry",
|
||||
"twml/runtime/src/main/scala/com/twitter/deepbird/runtime/prediction_engine",
|
||||
"util/util-slf4j-api/src/main/scala",
|
||||
],
|
||||
)
|
||||
@ -0,0 +1,20 @@
|
||||
package com.twitter.follow_recommendations.modules
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.follow_recommendations.configapi.ConfigBuilder
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.servo.decider.DeciderGateBuilder
|
||||
import com.twitter.timelines.configapi.Config
|
||||
import javax.inject.Singleton
|
||||
|
||||
object ConfigApiModule extends TwitterModule {
|
||||
@Provides
|
||||
@Singleton
|
||||
def providesDeciderGateBuilder(decider: Decider): DeciderGateBuilder =
|
||||
new DeciderGateBuilder(decider)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def providesConfig(configBuilder: ConfigBuilder): Config = configBuilder.build()
|
||||
}
|
||||
@ -0,0 +1,71 @@
|
||||
package com.twitter.follow_recommendations.modules
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.google.inject.Singleton
|
||||
import com.twitter.inject.annotations.Flag
|
||||
import com.twitter.decider.RandomRecipient
|
||||
import com.twitter.finagle.ThriftMux
|
||||
import com.twitter.finagle.mtls.authentication.ServiceIdentifier
|
||||
import com.twitter.finagle.mtls.client.MtlsStackClient.MtlsThriftMuxClientSyntax
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finagle.thrift.ClientId
|
||||
import com.twitter.finatra.annotations.DarkTrafficService
|
||||
import com.twitter.follow_recommendations.configapi.deciders.DeciderKey
|
||||
import com.twitter.follow_recommendations.thriftscala.FollowRecommendationsThriftService
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.inject.thrift.filters.DarkTrafficFilter
|
||||
import com.twitter.servo.decider.DeciderGateBuilder
|
||||
|
||||
object DiffyModule extends TwitterModule {
|
||||
// diffy.dest is defined in the Follow Recommendations Service aurora file
|
||||
// and points to the Dark Traffic Proxy server
|
||||
private val destFlag =
|
||||
flag[String]("diffy.dest", "/$/nil", "Resolvable name of diffy-service or proxy")
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@DarkTrafficService
|
||||
def provideDarkTrafficService(
|
||||
serviceIdentifier: ServiceIdentifier
|
||||
): FollowRecommendationsThriftService.ReqRepServicePerEndpoint = {
|
||||
ThriftMux.client
|
||||
.withClientId(ClientId("follow_recos_service_darktraffic_proxy_client"))
|
||||
.withMutualTls(serviceIdentifier)
|
||||
.servicePerEndpoint[FollowRecommendationsThriftService.ReqRepServicePerEndpoint](
|
||||
dest = destFlag(),
|
||||
label = "darktrafficproxy"
|
||||
)
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def provideDarkTrafficFilter(
|
||||
@DarkTrafficService darkService: FollowRecommendationsThriftService.ReqRepServicePerEndpoint,
|
||||
deciderGateBuilder: DeciderGateBuilder,
|
||||
statsReceiver: StatsReceiver,
|
||||
@Flag("environment") env: String
|
||||
): DarkTrafficFilter[FollowRecommendationsThriftService.ReqRepServicePerEndpoint] = {
|
||||
// sampleFunction is used to determine which requests should get replicated
|
||||
// to the dark traffic proxy server
|
||||
val sampleFunction: Any => Boolean = { _ =>
|
||||
// check whether the current FRS instance is deployed in production
|
||||
env match {
|
||||
case "prod" =>
|
||||
statsReceiver.scope("provideDarkTrafficFilter").counter("prod").incr()
|
||||
destFlag.isDefined && deciderGateBuilder
|
||||
.keyToFeature(DeciderKey.EnableTrafficDarkReading).isAvailable(RandomRecipient)
|
||||
case _ =>
|
||||
statsReceiver.scope("provideDarkTrafficFilter").counter("devel").incr()
|
||||
// replicate zero requests if in non-production environment
|
||||
false
|
||||
}
|
||||
}
|
||||
new DarkTrafficFilter[FollowRecommendationsThriftService.ReqRepServicePerEndpoint](
|
||||
darkService,
|
||||
sampleFunction,
|
||||
forwardAfterService = true,
|
||||
statsReceiver.scope("DarkTrafficFilter"),
|
||||
lookupByMethod = true
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,85 @@
|
||||
package com.twitter.follow_recommendations.modules
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.abdecider.LoggingABDecider
|
||||
import com.twitter.featureswitches.v2.Feature
|
||||
import com.twitter.featureswitches.v2.FeatureFilter
|
||||
import com.twitter.featureswitches.v2.FeatureSwitches
|
||||
import com.twitter.featureswitches.v2.builder.FeatureSwitchesBuilder
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants.PRODUCER_SIDE_FEATURE_SWITCHES
|
||||
import com.twitter.inject.TwitterModule
|
||||
import javax.inject.Named
|
||||
import javax.inject.Singleton
|
||||
|
||||
object FeaturesSwitchesModule extends TwitterModule {
|
||||
private val DefaultConfigRepoPath = "/usr/local/config"
|
||||
private val FeaturesPath = "/features/onboarding/follow-recommendations-service/main"
|
||||
val isLocal = flag("configrepo.local", false, "Is the server running locally or in a DC")
|
||||
val localConfigRepoPath = flag(
|
||||
"local.configrepo",
|
||||
System.getProperty("user.home") + "/workspace/config",
|
||||
"Path to your local config repo"
|
||||
)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def providesFeatureSwitches(
|
||||
abDecider: LoggingABDecider,
|
||||
statsReceiver: StatsReceiver
|
||||
): FeatureSwitches = {
|
||||
val configRepoPath = if (isLocal()) {
|
||||
localConfigRepoPath()
|
||||
} else {
|
||||
DefaultConfigRepoPath
|
||||
}
|
||||
|
||||
FeatureSwitchesBuilder
|
||||
.createDefault(FeaturesPath, abDecider, Some(statsReceiver))
|
||||
.configRepoAbsPath(configRepoPath)
|
||||
.serviceDetailsFromAurora()
|
||||
.build()
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(PRODUCER_SIDE_FEATURE_SWITCHES)
|
||||
def providesProducerFeatureSwitches(
|
||||
abDecider: LoggingABDecider,
|
||||
statsReceiver: StatsReceiver
|
||||
): FeatureSwitches = {
|
||||
val configRepoPath = if (isLocal()) {
|
||||
localConfigRepoPath()
|
||||
} else {
|
||||
DefaultConfigRepoPath
|
||||
}
|
||||
|
||||
/**
|
||||
* Feature Switches evaluate all tied FS Keys on Params construction time, which is very inefficient
|
||||
* for producer/candidate side holdbacks because we have 100s of candidates, and 100s of FS which result
|
||||
* in 10,000 FS evaluations when we want 1 per candidate (100 total), so we create a new FS Client
|
||||
* which has a [[ProducerFeatureFilter]] set for feature filter to reduce the FS Keys we evaluate.
|
||||
*/
|
||||
FeatureSwitchesBuilder
|
||||
.createDefault(FeaturesPath, abDecider, Some(statsReceiver.scope("producer_side_fs")))
|
||||
.configRepoAbsPath(configRepoPath)
|
||||
.serviceDetailsFromAurora()
|
||||
.addFeatureFilter(ProducerFeatureFilter)
|
||||
.build()
|
||||
}
|
||||
}
|
||||
|
||||
case object ProducerFeatureFilter extends FeatureFilter {
|
||||
private val AllowedKeys = Set(
|
||||
"post_nux_ml_flow_candidate_user_scorer_id",
|
||||
"frs_receiver_holdback_keep_social_user_candidate",
|
||||
"frs_receiver_holdback_keep_user_candidate")
|
||||
|
||||
override def filter(feature: Feature): Option[Feature] = {
|
||||
if (AllowedKeys.exists(feature.parameters.contains)) {
|
||||
Some(feature)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,18 @@
|
||||
package com.twitter.follow_recommendations.modules
|
||||
import com.twitter.inject.TwitterModule
|
||||
|
||||
object FlagsModule extends TwitterModule {
|
||||
flag[Boolean](
|
||||
name = "fetch_prod_promoted_accounts",
|
||||
help = "Whether or not to fetch production promoted accounts (true / false)"
|
||||
)
|
||||
flag[Boolean](
|
||||
name = "interests_opt_out_prod_enabled",
|
||||
help = "Whether to fetch intersts opt out data from the prod strato column or not"
|
||||
)
|
||||
flag[Boolean](
|
||||
name = "log_results",
|
||||
default = false,
|
||||
help = "Whether to log results such that we use them for scoring or metrics"
|
||||
)
|
||||
}
|
||||
@ -0,0 +1,12 @@
|
||||
package com.twitter.follow_recommendations.modules
|
||||
|
||||
import com.twitter.follow_recommendations.products.ProdProductRegistry
|
||||
import com.twitter.follow_recommendations.products.common.ProductRegistry
|
||||
import com.twitter.inject.TwitterModule
|
||||
import javax.inject.Singleton
|
||||
|
||||
object ProductRegistryModule extends TwitterModule {
|
||||
override protected def configure(): Unit = {
|
||||
bind[ProductRegistry].to[ProdProductRegistry].in[Singleton]
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,40 @@
|
||||
package com.twitter.follow_recommendations.modules
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.google.inject.Singleton
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.relevance.ep_model.common.CommonConstants
|
||||
import com.twitter.relevance.ep_model.scorer.EPScorer
|
||||
import com.twitter.relevance.ep_model.scorer.EPScorerBuilder
|
||||
import java.io.File
|
||||
import java.io.FileOutputStream
|
||||
import scala.language.postfixOps
|
||||
|
||||
object ScorerModule extends TwitterModule {
|
||||
private val STPScorerPath = "/quality/stp_models/20141223"
|
||||
|
||||
private def fileFromResource(resource: String): File = {
|
||||
val inputStream = getClass.getResourceAsStream(resource)
|
||||
val file = File.createTempFile(resource, "temp")
|
||||
val fos = new FileOutputStream(file)
|
||||
Iterator
|
||||
.continually(inputStream.read)
|
||||
.takeWhile(-1 !=)
|
||||
.foreach(fos.write)
|
||||
file
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
def provideEpScorer: EPScorer = {
|
||||
val modelPath =
|
||||
fileFromResource(STPScorerPath + "/" + CommonConstants.EP_MODEL_FILE_NAME).getAbsolutePath
|
||||
val trainingConfigPath =
|
||||
fileFromResource(STPScorerPath + "/" + CommonConstants.TRAINING_CONFIG).getAbsolutePath
|
||||
val epScorer = new EPScorerBuilder
|
||||
epScorer
|
||||
.withModelPath(modelPath)
|
||||
.withTrainingConfig(trainingConfigPath)
|
||||
.build()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,95 @@
|
||||
package com.twitter.follow_recommendations.modules
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.google.inject.Singleton
|
||||
import com.google.inject.name.Named
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.follow_recommendations.common.constants.GuiceNamedConstants
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.logging.BareFormatter
|
||||
import com.twitter.logging.HandlerFactory
|
||||
import com.twitter.logging.Level
|
||||
import com.twitter.logging.LoggerFactory
|
||||
import com.twitter.logging.NullHandler
|
||||
import com.twitter.logging.QueueingHandler
|
||||
import com.twitter.logging.ScribeHandler
|
||||
|
||||
object ScribeModule extends TwitterModule {
|
||||
val useProdLogger = flag(
|
||||
name = "scribe.use_prod_loggers",
|
||||
default = false,
|
||||
help = "whether to use production logging for service"
|
||||
)
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.CLIENT_EVENT_LOGGER)
|
||||
def provideClientEventsLoggerFactory(stats: StatsReceiver): LoggerFactory = {
|
||||
val loggerCategory = "client_event"
|
||||
val clientEventsHandler: HandlerFactory = if (useProdLogger()) {
|
||||
QueueingHandler(
|
||||
maxQueueSize = 10000,
|
||||
handler = ScribeHandler(
|
||||
category = loggerCategory,
|
||||
formatter = BareFormatter,
|
||||
level = Some(Level.INFO),
|
||||
statsReceiver = stats.scope("client_event_scribe")
|
||||
)
|
||||
)
|
||||
} else { () => NullHandler }
|
||||
LoggerFactory(
|
||||
node = "abdecider",
|
||||
level = Some(Level.INFO),
|
||||
useParents = false,
|
||||
handlers = clientEventsHandler :: Nil
|
||||
)
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.REQUEST_LOGGER)
|
||||
def provideFollowRecommendationsLoggerFactory(stats: StatsReceiver): LoggerFactory = {
|
||||
val loggerCategory = "follow_recommendations_logs"
|
||||
val handlerFactory: HandlerFactory = if (useProdLogger()) {
|
||||
QueueingHandler(
|
||||
maxQueueSize = 10000,
|
||||
handler = ScribeHandler(
|
||||
category = loggerCategory,
|
||||
formatter = BareFormatter,
|
||||
level = Some(Level.INFO),
|
||||
statsReceiver = stats.scope("follow_recommendations_logs_scribe")
|
||||
)
|
||||
)
|
||||
} else { () => NullHandler }
|
||||
LoggerFactory(
|
||||
node = loggerCategory,
|
||||
level = Some(Level.INFO),
|
||||
useParents = false,
|
||||
handlers = handlerFactory :: Nil
|
||||
)
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
@Named(GuiceNamedConstants.FLOW_LOGGER)
|
||||
def provideFrsRecommendationFlowLoggerFactory(stats: StatsReceiver): LoggerFactory = {
|
||||
val loggerCategory = "frs_recommendation_flow_logs"
|
||||
val handlerFactory: HandlerFactory = if (useProdLogger()) {
|
||||
QueueingHandler(
|
||||
maxQueueSize = 10000,
|
||||
handler = ScribeHandler(
|
||||
category = loggerCategory,
|
||||
formatter = BareFormatter,
|
||||
level = Some(Level.INFO),
|
||||
statsReceiver = stats.scope("frs_recommendation_flow_logs_scribe")
|
||||
)
|
||||
)
|
||||
} else { () => NullHandler }
|
||||
LoggerFactory(
|
||||
node = loggerCategory,
|
||||
level = Some(Level.INFO),
|
||||
useParents = false,
|
||||
handlers = handlerFactory :: Nil
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,13 @@
|
||||
package com.twitter.follow_recommendations.modules
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.google.inject.Singleton
|
||||
import com.twitter.finagle.memcached.ZookeeperStateMonitor.DefaultTimer
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.util.Timer
|
||||
|
||||
object TimerModule extends TwitterModule {
|
||||
@Provides
|
||||
@Singleton
|
||||
def providesTimer: Timer = DefaultTimer
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user