Track trending tags (#7638)
* Track trending tags - Half-life of 1 day - Historical usage in daily buckets (last 7 days stored) - GET /api/v1/trends Fix #271 * Add trends to web UI * Don't render compose form on search route, adjust search results header * Disqualify tag from trends if it's in disallowed hashtags setting * Count distinct accounts using tag, ignore silenced accounts
This commit is contained in:
Eugen Rochko
GitHub
parent
63c7b91572
commit
9bd23dc4e5
|
|
@ -0,0 +1,17 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class Api::V1::TrendsController < Api::BaseController
|
||||
before_action :set_tags
|
||||
|
||||
respond_to :json
|
||||
|
||||
def index
|
||||
render json: @tags, each_serializer: REST::TagSerializer
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def set_tags
|
||||
@tags = TrendingTags.get(limit_param(10))
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
import api from '../api';
|
||||
|
||||
export const TRENDS_FETCH_REQUEST = 'TRENDS_FETCH_REQUEST';
|
||||
export const TRENDS_FETCH_SUCCESS = 'TRENDS_FETCH_SUCCESS';
|
||||
export const TRENDS_FETCH_FAIL = 'TRENDS_FETCH_FAIL';
|
||||
|
||||
export const fetchTrends = () => (dispatch, getState) => {
|
||||
dispatch(fetchTrendsRequest());
|
||||
|
||||
api(getState)
|
||||
.get('/api/v1/trends')
|
||||
.then(({ data }) => dispatch(fetchTrendsSuccess(data)))
|
||||
.catch(err => dispatch(fetchTrendsFail(err)));
|
||||
};
|
||||
|
||||
export const fetchTrendsRequest = () => ({
|
||||
type: TRENDS_FETCH_REQUEST,
|
||||
skipLoading: true,
|
||||
});
|
||||
|
||||
export const fetchTrendsSuccess = trends => ({
|
||||
type: TRENDS_FETCH_SUCCESS,
|
||||
trends,
|
||||
skipLoading: true,
|
||||
});
|
||||
|
||||
export const fetchTrendsFail = error => ({
|
||||
type: TRENDS_FETCH_FAIL,
|
||||
error,
|
||||
skipLoading: true,
|
||||
skipAlert: true,
|
||||
});
|
||||
|
|
@ -1,23 +1,75 @@
|
|||
import React from 'react';
|
||||
import PropTypes from 'prop-types';
|
||||
import ImmutablePropTypes from 'react-immutable-proptypes';
|
||||
import { FormattedMessage } from 'react-intl';
|
||||
import { FormattedMessage, FormattedNumber } from 'react-intl';
|
||||
import AccountContainer from '../../../containers/account_container';
|
||||
import StatusContainer from '../../../containers/status_container';
|
||||
import { Link } from 'react-router-dom';
|
||||
import ImmutablePureComponent from 'react-immutable-pure-component';
|
||||
import { Sparklines, SparklinesCurve } from 'react-sparklines';
|
||||
|
||||
const shortNumberFormat = number => {
|
||||
if (number < 1000) {
|
||||
return <FormattedNumber value={number} />;
|
||||
} else {
|
||||
return <React.Fragment><FormattedNumber value={number / 1000} maximumFractionDigits={1} />K</React.Fragment>;
|
||||
}
|
||||
};
|
||||
|
||||
export default class SearchResults extends ImmutablePureComponent {
|
||||
|
||||
static propTypes = {
|
||||
results: ImmutablePropTypes.map.isRequired,
|
||||
trends: ImmutablePropTypes.list,
|
||||
fetchTrends: PropTypes.func.isRequired,
|
||||
};
|
||||
|
||||
componentDidMount () {
|
||||
const { fetchTrends } = this.props;
|
||||
fetchTrends();
|
||||
}
|
||||
|
||||
render () {
|
||||
const { results } = this.props;
|
||||
const { results, trends } = this.props;
|
||||
|
||||
let accounts, statuses, hashtags;
|
||||
let count = 0;
|
||||
|
||||
if (results.isEmpty()) {
|
||||
return (
|
||||
<div className='search-results'>
|
||||
<div className='trends'>
|
||||
<div className='trends__header'>
|
||||
<i className='fa fa-fire fa-fw' />
|
||||
<FormattedMessage id='trends.header' defaultMessage='Trending now' />
|
||||
</div>
|
||||
|
||||
{trends && trends.map(hashtag => (
|
||||
<div className='trends__item' key={hashtag.get('name')}>
|
||||
<div className='trends__item__name'>
|
||||
<Link to={`/timelines/tag/${hashtag.get('name')}`}>
|
||||
#<span>{hashtag.get('name')}</span>
|
||||
</Link>
|
||||
|
||||
<FormattedMessage id='trends.count_by_accounts' defaultMessage='{count} {rawCount, plural, one {person} other {people}} talking' values={{ rawCount: hashtag.getIn(['history', 0, 'accounts']), count: <strong>{shortNumberFormat(hashtag.getIn(['history', 0, 'accounts']))}</strong> }} />
|
||||
</div>
|
||||
|
||||
<div className='trends__item__current'>
|
||||
{shortNumberFormat(hashtag.getIn(['history', 0, 'uses']))}
|
||||
</div>
|
||||
|
||||
<div className='trends__item__sparkline'>
|
||||
<Sparklines width={50} height={28} data={hashtag.get('history').reverse().map(day => day.get('uses')).toArray()}>
|
||||
<SparklinesCurve style={{ fill: 'none' }} />
|
||||
</Sparklines>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (results.get('accounts') && results.get('accounts').size > 0) {
|
||||
count += results.get('accounts').size;
|
||||
accounts = (
|
||||
|
|
@ -48,7 +100,7 @@ export default class SearchResults extends ImmutablePureComponent {
|
|||
|
||||
{results.get('hashtags').map(hashtag => (
|
||||
<Link key={hashtag} className='search-results__hashtag' to={`/timelines/tag/${hashtag}`}>
|
||||
#{hashtag}
|
||||
{hashtag}
|
||||
</Link>
|
||||
))}
|
||||
</div>
|
||||
|
|
@ -58,6 +110,7 @@ export default class SearchResults extends ImmutablePureComponent {
|
|||
return (
|
||||
<div className='search-results'>
|
||||
<div className='search-results__header'>
|
||||
<i className='fa fa-search fa-fw' />
|
||||
<FormattedMessage id='search_results.total' defaultMessage='{count, number} {count, plural, one {result} other {results}}' values={{ count }} />
|
||||
</div>
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,14 @@
|
|||
import { connect } from 'react-redux';
|
||||
import SearchResults from '../components/search_results';
|
||||
import { fetchTrends } from '../../../actions/trends';
|
||||
|
||||
const mapStateToProps = state => ({
|
||||
results: state.getIn(['search', 'results']),
|
||||
trends: state.get('trends'),
|
||||
});
|
||||
|
||||
export default connect(mapStateToProps)(SearchResults);
|
||||
const mapDispatchToProps = dispatch => ({
|
||||
fetchTrends: () => dispatch(fetchTrends()),
|
||||
});
|
||||
|
||||
export default connect(mapStateToProps, mapDispatchToProps)(SearchResults);
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ export default class Compose extends React.PureComponent {
|
|||
{(multiColumn || isSearchPage) && <SearchContainer /> }
|
||||
|
||||
<div className='drawer__pager'>
|
||||
<div className='drawer__inner' onFocus={this.onFocus}>
|
||||
{!isSearchPage && <div className='drawer__inner' onFocus={this.onFocus}>
|
||||
<NavigationContainer onClose={this.onBlur} />
|
||||
<ComposeFormContainer />
|
||||
{multiColumn && (
|
||||
|
|
@ -109,7 +109,7 @@ export default class Compose extends React.PureComponent {
|
|||
<img alt='' draggable='false' src={elephantUIPlane} />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>}
|
||||
|
||||
<Motion defaultStyle={{ x: isSearchPage ? 0 : -100 }} style={{ x: spring(showSearch || isSearchPage ? 0 : -100, { stiffness: 210, damping: 20 }) }}>
|
||||
{({ x }) => (
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import height_cache from './height_cache';
|
|||
import custom_emojis from './custom_emojis';
|
||||
import lists from './lists';
|
||||
import listEditor from './list_editor';
|
||||
import trends from './trends';
|
||||
|
||||
const reducers = {
|
||||
dropdown_menu,
|
||||
|
|
@ -55,6 +56,7 @@ const reducers = {
|
|||
custom_emojis,
|
||||
lists,
|
||||
listEditor,
|
||||
trends,
|
||||
};
|
||||
|
||||
export default combineReducers(reducers);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,13 @@
|
|||
import { TRENDS_FETCH_SUCCESS } from '../actions/trends';
|
||||
import { fromJS } from 'immutable';
|
||||
|
||||
const initialState = null;
|
||||
|
||||
export default function trendsReducer(state = initialState, action) {
|
||||
switch(action.type) {
|
||||
case TRENDS_FETCH_SUCCESS:
|
||||
return fromJS(action.trends);
|
||||
default:
|
||||
return state;
|
||||
}
|
||||
};
|
||||
|
|
@ -3334,9 +3334,15 @@ a.status-card {
|
|||
color: $dark-text-color;
|
||||
background: lighten($ui-base-color, 2%);
|
||||
border-bottom: 1px solid darken($ui-base-color, 4%);
|
||||
padding: 15px 10px;
|
||||
font-size: 14px;
|
||||
padding: 15px;
|
||||
font-weight: 500;
|
||||
font-size: 16px;
|
||||
cursor: default;
|
||||
|
||||
.fa {
|
||||
display: inline-block;
|
||||
margin-right: 5px;
|
||||
}
|
||||
}
|
||||
|
||||
.search-results__section {
|
||||
|
|
@ -5209,3 +5215,76 @@ noscript {
|
|||
background: $ui-base-color;
|
||||
}
|
||||
}
|
||||
|
||||
.trends {
|
||||
&__header {
|
||||
color: $dark-text-color;
|
||||
background: lighten($ui-base-color, 2%);
|
||||
border-bottom: 1px solid darken($ui-base-color, 4%);
|
||||
font-weight: 500;
|
||||
padding: 15px;
|
||||
font-size: 16px;
|
||||
cursor: default;
|
||||
|
||||
.fa {
|
||||
display: inline-block;
|
||||
margin-right: 5px;
|
||||
}
|
||||
}
|
||||
|
||||
&__item {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
padding: 15px;
|
||||
border-bottom: 1px solid lighten($ui-base-color, 8%);
|
||||
|
||||
&:last-child {
|
||||
border-bottom: 0;
|
||||
}
|
||||
|
||||
&__name {
|
||||
flex: 1 1 auto;
|
||||
color: $dark-text-color;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
|
||||
strong {
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
a {
|
||||
color: $darker-text-color;
|
||||
text-decoration: none;
|
||||
font-size: 14px;
|
||||
font-weight: 500;
|
||||
display: block;
|
||||
|
||||
&:hover,
|
||||
&:focus,
|
||||
&:active {
|
||||
span {
|
||||
text-decoration: underline;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
&__current {
|
||||
width: 100px;
|
||||
font-size: 24px;
|
||||
line-height: 36px;
|
||||
font-weight: 500;
|
||||
text-align: center;
|
||||
color: $secondary-text-color;
|
||||
}
|
||||
|
||||
&__sparkline {
|
||||
width: 50px;
|
||||
|
||||
path {
|
||||
stroke: lighten($highlight-text-color, 6%) !important;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,6 +21,22 @@ class Tag < ApplicationRecord
|
|||
name
|
||||
end
|
||||
|
||||
def history
|
||||
days = []
|
||||
|
||||
7.times do |i|
|
||||
day = i.days.ago.beginning_of_day.to_i
|
||||
|
||||
days << {
|
||||
day: day.to_s,
|
||||
uses: Redis.current.get("activity:tags:#{id}:#{day}") || '0',
|
||||
accounts: Redis.current.pfcount("activity:tags:#{id}:#{day}:accounts").to_s,
|
||||
}
|
||||
end
|
||||
|
||||
days
|
||||
end
|
||||
|
||||
class << self
|
||||
def search_for(term, limit = 5)
|
||||
pattern = sanitize_sql_like(term.strip) + '%'
|
||||
|
|
|
|||
|
|
@ -0,0 +1,61 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class TrendingTags
|
||||
KEY = 'trending_tags'
|
||||
HALF_LIFE = 1.day.to_i
|
||||
MAX_ITEMS = 500
|
||||
EXPIRE_HISTORY_AFTER = 7.days.seconds
|
||||
|
||||
class << self
|
||||
def record_use!(tag, account, at_time = Time.now.utc)
|
||||
return if disallowed_hashtags.include?(tag.name) || account.silenced?
|
||||
|
||||
increment_vote!(tag.id, at_time)
|
||||
increment_historical_use!(tag.id, at_time)
|
||||
increment_unique_use!(tag.id, account.id, at_time)
|
||||
end
|
||||
|
||||
def get(limit)
|
||||
tag_ids = redis.zrevrange(KEY, 0, limit).map(&:to_i)
|
||||
tags = Tag.where(id: tag_ids).to_a.map { |tag| [tag.id, tag] }.to_h
|
||||
tag_ids.map { |tag_id| tags[tag_id] }.compact
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def increment_vote!(tag_id, at_time)
|
||||
redis.zincrby(KEY, (2**((at_time.to_i - epoch) / HALF_LIFE)).to_f, tag_id.to_s)
|
||||
redis.zremrangebyrank(KEY, 0, -MAX_ITEMS) if rand < (2.to_f / MAX_ITEMS)
|
||||
end
|
||||
|
||||
def increment_historical_use!(tag_id, at_time)
|
||||
key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}"
|
||||
redis.incrby(key, 1)
|
||||
redis.expire(key, EXPIRE_HISTORY_AFTER)
|
||||
end
|
||||
|
||||
def increment_unique_use!(tag_id, account_id, at_time)
|
||||
key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts"
|
||||
redis.pfadd(key, account_id)
|
||||
redis.expire(key, EXPIRE_HISTORY_AFTER)
|
||||
end
|
||||
|
||||
# The epoch needs to be 2.5 years in the future if the half-life is one day
|
||||
# While dynamic, it will always be the same within one year
|
||||
def epoch
|
||||
@epoch ||= Date.new(Date.current.year + 2.5, 10, 1).to_datetime.to_i
|
||||
end
|
||||
|
||||
def disallowed_hashtags
|
||||
return @disallowed_hashtags if defined?(@disallowed_hashtags)
|
||||
|
||||
@disallowed_hashtags = Setting.disallowed_hashtags.nil? ? [] : Setting.disallowed_hashtags
|
||||
@disallowed_hashtags = @disallowed_hashtags.split(' ') if @disallowed_hashtags.is_a? String
|
||||
@disallowed_hashtags = @disallowed_hashtags.map(&:downcase)
|
||||
end
|
||||
|
||||
def redis
|
||||
Redis.current
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
class REST::TagSerializer < ActiveModel::Serializer
|
||||
include RoutingHelper
|
||||
|
||||
attributes :name, :url, :history
|
||||
|
||||
def url
|
||||
tag_url(object)
|
||||
end
|
||||
end
|
||||
|
|
@ -4,8 +4,10 @@ class ProcessHashtagsService < BaseService
|
|||
def call(status, tags = [])
|
||||
tags = Extractor.extract_hashtags(status.text) if status.local?
|
||||
|
||||
tags.map { |str| str.mb_chars.downcase }.uniq(&:to_s).each do |tag|
|
||||
status.tags << Tag.where(name: tag).first_or_initialize(name: tag)
|
||||
tags.map { |str| str.mb_chars.downcase }.uniq(&:to_s).each do |name|
|
||||
tag = Tag.where(name: name).first_or_create(name: name)
|
||||
status.tags << tag
|
||||
TrendingTags.record_use!(tag, status.account, status.created_at)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -254,6 +254,7 @@ Rails.application.routes.draw do
|
|||
resources :mutes, only: [:index]
|
||||
resources :favourites, only: [:index]
|
||||
resources :reports, only: [:index, :create]
|
||||
resources :trends, only: [:index]
|
||||
|
||||
namespace :apps do
|
||||
get :verify_credentials, to: 'credentials#show'
|
||||
|
|
|
|||
|
|
@ -97,6 +97,7 @@
|
|||
"react-redux-loading-bar": "^2.9.3",
|
||||
"react-router-dom": "^4.1.1",
|
||||
"react-router-scroll-4": "^1.0.0-beta.1",
|
||||
"react-sparklines": "^1.7.0",
|
||||
"react-swipeable-views": "^0.12.3",
|
||||
"react-textarea-autosize": "^5.2.1",
|
||||
"react-toggle": "^4.0.1",
|
||||
|
|
|
|||
|
|
@ -6124,6 +6124,12 @@ react-router@^4.2.0:
|
|||
prop-types "^15.5.4"
|
||||
warning "^3.0.0"
|
||||
|
||||
react-sparklines@^1.7.0:
|
||||
version "1.7.0"
|
||||
resolved "https://registry.yarnpkg.com/react-sparklines/-/react-sparklines-1.7.0.tgz#9b1d97e8c8610095eeb2ad658d2e1fcf91f91a60"
|
||||
dependencies:
|
||||
prop-types "^15.5.10"
|
||||
|
||||
react-swipeable-views-core@^0.12.11:
|
||||
version "0.12.11"
|
||||
resolved "https://registry.yarnpkg.com/react-swipeable-views-core/-/react-swipeable-views-core-0.12.11.tgz#3cf2b4daffbb36f9d69bd19bf5b2d5370b6b2c1b"
|
||||
|
|
|
|||
Loading…
Reference in New Issue