Twitter Recommendation Algorithm

Please note we have force-pushed a new initial commit in order to remove some publicly-available Twitter user information. Note that this process may be required in the future.

timelineranker/client/builder/BUILD

@@ -0,0 +1,6 @@
+target(
+    tags = ["bazel-compatible"],
+    dependencies = [
+        "timelineranker/client/builder/src/main/scala",
+    ],
+)

timelineranker/client/builder/README.md

@@ -0,0 +1,4 @@
+# TimelineRanker client
+
+Library for creating a client to talk to TLR. It contains a ClientBuilder implementation
+with some preferred settings for clients.

timelineranker/client/builder/src/main/scala/BUILD

@@ -0,0 +1,16 @@
+scala_library(
+    sources = ["com/twitter/timelineranker/client/*.scala"],
+    platform = "java8",
+    tags = ["bazel-compatible"],
+    dependencies = [
+        "finagle/finagle-core/src/main",
+        "finagle/finagle-stats",
+        "finagle/finagle-thrift/src/main/java",
+        "servo/client/src/main/scala/com/twitter/servo/client",
+        "src/thrift/com/twitter/timelineranker:thrift-scala",
+        "src/thrift/com/twitter/timelineranker/server/model:thrift-scala",
+        "timelineranker/common:model",
+        "timelines/src/main/scala/com/twitter/timelines/util/stats",
+        "util/util-stats/src/main/scala",
+    ],
+)

timelineranker/client/builder/src/main/scala/com/twitter/timelineranker/client/TimelineRankerClient.scala

@@ -0,0 +1,195 @@
+package com.twitter.timelineranker.client
+
+import com.twitter.finagle.SourcedException
+import com.twitter.finagle.stats.StatsReceiver
+import com.twitter.timelineranker.{thriftscala => thrift}
+import com.twitter.timelineranker.model._
+import com.twitter.timelines.util.stats.RequestStats
+import com.twitter.timelines.util.stats.RequestStatsReceiver
+import com.twitter.util.Future
+import com.twitter.util.Return
+import com.twitter.util.Throw
+import com.twitter.util.Try
+
+case class TimelineRankerException(message: String)
+    extends Exception(message)
+    with SourcedException {
+  serviceName = "timelineranker"
+}
+
+/**
+ * A timeline ranker client whose methods accept and produce model object instances
+ * instead of thrift instances.
+ */
+class TimelineRankerClient(
+  private val client: thrift.TimelineRanker.MethodPerEndpoint,
+  statsReceiver: StatsReceiver)
+    extends RequestStats {
+
+  private[this] val baseScope = statsReceiver.scope("timelineRankerClient")
+  private[this] val timelinesRequestStats = RequestStatsReceiver(baseScope.scope("timelines"))
+  private[this] val recycledTweetRequestStats = RequestStatsReceiver(
+    baseScope.scope("recycledTweet"))
+  private[this] val recapHydrationRequestStats = RequestStatsReceiver(
+    baseScope.scope("recapHydration"))
+  private[this] val recapAuthorRequestStats = RequestStatsReceiver(baseScope.scope("recapAuthor"))
+  private[this] val entityTweetsRequestStats = RequestStatsReceiver(baseScope.scope("entityTweets"))
+  private[this] val utegLikedByTweetsRequestStats = RequestStatsReceiver(
+    baseScope.scope("utegLikedByTweets"))
+
+  private[this] def fetchRecapQueryResultHead(
+    results: Seq[Try[CandidateTweetsResult]]
+  ): CandidateTweetsResult = {
+    results.head match {
+      case Return(result) => result
+      case Throw(e) => throw e
+    }
+  }
+
+  private[this] def tryResults[Req, Rep](
+    reqs: Seq[Req],
+    stats: RequestStatsReceiver,
+    findError: Req => Option[thrift.TimelineError],
+  )(
+    getRep: (Req, RequestStatsReceiver) => Try[Rep]
+  ): Seq[Try[Rep]] = {
+    reqs.map { req =>
+      findError(req) match {
+        case Some(error) if error.reason.exists { _ == thrift.ErrorReason.OverCapacity } =>
+          // bubble up over capacity error, server shall handle it
+          stats.onFailure(error)
+          Throw(error)
+        case Some(error) =>
+          stats.onFailure(error)
+          Throw(TimelineRankerException(error.message))
+        case None =>
+          getRep(req, stats)
+      }
+    }
+  }
+
+  private[this] def tryCandidateTweetsResults(
+    responses: Seq[thrift.GetCandidateTweetsResponse],
+    requestScopedStats: RequestStatsReceiver
+  ): Seq[Try[CandidateTweetsResult]] = {
+    def errorInResponse(
+      response: thrift.GetCandidateTweetsResponse
+    ): Option[thrift.TimelineError] = {
+      response.error
+    }
+
+    tryResults(
+      responses,
+      requestScopedStats,
+      errorInResponse
+    ) { (response, stats) =>
+      stats.onSuccess()
+      Return(CandidateTweetsResult.fromThrift(response))
+    }
+  }
+
+  def getTimeline(query: TimelineQuery): Future[Try[Timeline]] = {
+    getTimelines(Seq(query)).map(_.head)
+  }
+
+  def getTimelines(queries: Seq[TimelineQuery]): Future[Seq[Try[Timeline]]] = {
+    def errorInResponse(response: thrift.GetTimelineResponse): Option[thrift.TimelineError] = {
+      response.error
+    }
+    val thriftQueries = queries.map(_.toThrift)
+    timelinesRequestStats.latency {
+      client.getTimelines(thriftQueries).map { responses =>
+        tryResults(
+          responses,
+          timelinesRequestStats,
+          errorInResponse
+        ) { (response, stats) =>
+          response.timeline match {
+            case Some(timeline) =>
+              stats.onSuccess()
+              Return(Timeline.fromThrift(timeline))
+            // Should not really happen.
+            case None =>
+              val tlrException =
+                TimelineRankerException("No timeline returned even when no error occurred.")
+              stats.onFailure(tlrException)
+              Throw(tlrException)
+          }
+        }
+      }
+    }
+  }
+
+  def getRecycledTweetCandidates(query: RecapQuery): Future[CandidateTweetsResult] = {
+    getRecycledTweetCandidates(Seq(query)).map(fetchRecapQueryResultHead)
+  }
+
+  def getRecycledTweetCandidates(
+    queries: Seq[RecapQuery]
+  ): Future[Seq[Try[CandidateTweetsResult]]] = {
+    val thriftQueries = queries.map(_.toThriftRecapQuery)
+    recycledTweetRequestStats.latency {
+      client.getRecycledTweetCandidates(thriftQueries).map {
+        tryCandidateTweetsResults(_, recycledTweetRequestStats)
+      }
+    }
+  }
+
+  def hydrateTweetCandidates(query: RecapQuery): Future[CandidateTweetsResult] = {
+    hydrateTweetCandidates(Seq(query)).map(fetchRecapQueryResultHead)
+  }
+
+  def hydrateTweetCandidates(queries: Seq[RecapQuery]): Future[Seq[Try[CandidateTweetsResult]]] = {
+    val thriftQueries = queries.map(_.toThriftRecapHydrationQuery)
+    recapHydrationRequestStats.latency {
+      client.hydrateTweetCandidates(thriftQueries).map {
+        tryCandidateTweetsResults(_, recapHydrationRequestStats)
+      }
+    }
+  }
+
+  def getRecapCandidatesFromAuthors(query: RecapQuery): Future[CandidateTweetsResult] = {
+    getRecapCandidatesFromAuthors(Seq(query)).map(fetchRecapQueryResultHead)
+  }
+
+  def getRecapCandidatesFromAuthors(
+    queries: Seq[RecapQuery]
+  ): Future[Seq[Try[CandidateTweetsResult]]] = {
+    val thriftQueries = queries.map(_.toThriftRecapQuery)
+    recapAuthorRequestStats.latency {
+      client.getRecapCandidatesFromAuthors(thriftQueries).map {
+        tryCandidateTweetsResults(_, recapAuthorRequestStats)
+      }
+    }
+  }
+
+  def getEntityTweetCandidates(query: RecapQuery): Future[CandidateTweetsResult] = {
+    getEntityTweetCandidates(Seq(query)).map(fetchRecapQueryResultHead)
+  }
+
+  def getEntityTweetCandidates(
+    queries: Seq[RecapQuery]
+  ): Future[Seq[Try[CandidateTweetsResult]]] = {
+    val thriftQueries = queries.map(_.toThriftEntityTweetsQuery)
+    entityTweetsRequestStats.latency {
+      client.getEntityTweetCandidates(thriftQueries).map {
+        tryCandidateTweetsResults(_, entityTweetsRequestStats)
+      }
+    }
+  }
+
+  def getUtegLikedByTweetCandidates(query: RecapQuery): Future[CandidateTweetsResult] = {
+    getUtegLikedByTweetCandidates(Seq(query)).map(fetchRecapQueryResultHead)
+  }
+
+  def getUtegLikedByTweetCandidates(
+    queries: Seq[RecapQuery]
+  ): Future[Seq[Try[CandidateTweetsResult]]] = {
+    val thriftQueries = queries.map(_.toThriftUtegLikedByTweetsQuery)
+    utegLikedByTweetsRequestStats.latency {
+      client.getUtegLikedByTweetCandidates(thriftQueries).map {
+        tryCandidateTweetsResults(_, utegLikedByTweetsRequestStats)
+      }
+    }
+  }
+}

timelineranker/client/builder/src/main/scala/com/twitter/timelineranker/client/TimelineRankerClientBuilder.scala

@@ -0,0 +1,89 @@
+package com.twitter.timelineranker.client
+
+import com.twitter.conversions.DurationOps._
+import com.twitter.finagle.builder.ClientBuilder
+import com.twitter.finagle.mtls.authentication.EmptyServiceIdentifier
+import com.twitter.finagle.mtls.authentication.ServiceIdentifier
+import com.twitter.finagle.mtls.client.MtlsClientBuilder._
+import com.twitter.finagle.param.OppTls
+import com.twitter.finagle.service.RetryPolicy
+import com.twitter.finagle.service.RetryPolicy._
+import com.twitter.finagle.ssl.OpportunisticTls
+import com.twitter.finagle.thrift.ThriftClientRequest
+import com.twitter.servo.client.Environment.Local
+import com.twitter.servo.client.Environment.Staging
+import com.twitter.servo.client.Environment.Production
+import com.twitter.servo.client.Environment
+import com.twitter.servo.client.FinagleClientBuilder
+import com.twitter.util.Try
+import com.twitter.util.Duration
+
+sealed trait TimelineRankerClientBuilderBase {
+  def DefaultName: String = "timelineranker"
+
+  def DefaultProdDest: String
+
+  def DefaultProdRequestTimeout: Duration = 2.seconds
+  def DefaultProdTimeout: Duration = 3.seconds
+  def DefaultProdRetryPolicy: RetryPolicy[Try[Nothing]] =
+    tries(2, TimeoutAndWriteExceptionsOnly orElse ChannelClosedExceptionsOnly)
+
+  def DefaultLocalTcpConnectTimeout: Duration = 1.second
+  def DefaultLocalConnectTimeout: Duration = 1.second
+  def DefaultLocalRetryPolicy: RetryPolicy[Try[Nothing]] = tries(2, TimeoutAndWriteExceptionsOnly)
+
+  def apply(
+    finagleClientBuilder: FinagleClientBuilder,
+    environment: Environment,
+    name: String = DefaultName,
+    serviceIdentifier: ServiceIdentifier = EmptyServiceIdentifier,
+    opportunisticTlsOpt: Option[OpportunisticTls.Level] = None,
+  ): ClientBuilder.Complete[ThriftClientRequest, Array[Byte]] = {
+    val defaultBuilder = finagleClientBuilder.thriftMuxClientBuilder(name)
+    val destination = getDestOverride(environment)
+
+    val partialClient = environment match {
+      case Production | Staging =>
+        defaultBuilder
+          .requestTimeout(DefaultProdRequestTimeout)
+          .timeout(DefaultProdTimeout)
+          .retryPolicy(DefaultProdRetryPolicy)
+          .daemon(daemonize = true)
+          .dest(destination)
+          .mutualTls(serviceIdentifier)
+      case Local =>
+        defaultBuilder
+          .tcpConnectTimeout(DefaultLocalTcpConnectTimeout)
+          .connectTimeout(DefaultLocalConnectTimeout)
+          .retryPolicy(DefaultLocalRetryPolicy)
+          .failFast(enabled = false)
+          .daemon(daemonize = false)
+          .dest(destination)
+          .mutualTls(serviceIdentifier)
+    }
+
+    opportunisticTlsOpt match {
+      case Some(_) =>
+        val opportunisticTlsParam = OppTls(level = opportunisticTlsOpt)
+        partialClient
+          .configured(opportunisticTlsParam)
+      case None => partialClient
+    }
+  }
+
+  private def getDestOverride(environment: Environment): String = {
+    val defaultDest = DefaultProdDest
+    environment match {
+      // Allow overriding the target TimelineRanker instance in staging.
+      // This is typically useful for redline testing of TimelineRanker.
+      case Staging =>
+        sys.props.getOrElse("target.timelineranker.instance", defaultDest)
+      case _ =>
+        defaultDest
+    }
+  }
+}
+
+object TimelineRankerClientBuilder extends TimelineRankerClientBuilderBase {
+  override def DefaultProdDest: String = "/s/timelineranker/timelineranker"
+}