diff options
| -rw-r--r-- | .travis.yml | 7 | ||||
| -rw-r--r-- | Dockerfile | 4 | ||||
| -rw-r--r-- | Gemfile | 2 | ||||
| -rw-r--r-- | Gemfile.lock | 6 | ||||
| -rw-r--r-- | Vagrantfile | 2 | ||||
| -rw-r--r-- | app/lib/language_detector.rb | 9 | ||||
| -rw-r--r-- | spec/lib/language_detector_spec.rb | 8 |
7 files changed, 23 insertions, 15 deletions
diff --git a/.travis.yml b/.travis.yml index 632074874..f18da476b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,9 +13,9 @@ env: - LOCAL_DOMAIN=cb6e6126.ngrok.io - LOCAL_HTTPS=true - RAILS_ENV=test - - CXX=g++-4.8 - NOKOGIRI_USE_SYSTEM_LIBRARIES=true - PARALLEL_TEST_PROCESSORS=2 + - "PATH=$HOME:$PATH" addons: postgresql: 9.4 @@ -24,8 +24,10 @@ addons: - ubuntu-toolchain-r-test - trusty-media packages: - - g++-4.8 - ffmpeg + - g++-6 + - libprotobuf-dev + - protobuf-compiler rvm: - 2.3.4 @@ -43,6 +45,7 @@ install: before_script: - bundle exec rake parallel:create parallel:load_schema parallel:prepare - bundle exec rails assets:precompile + - ln -s /usr/bin/x86_64-linux-gnu-g++-6 "$HOME/g++" script: - bundle exec parallel_test spec/ --group-by filesize --type rspec diff --git a/Dockerfile b/Dockerfile index 3e4727258..e2926725a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,8 @@ RUN echo "@edge https://nl.alpinelinux.org/alpine/edge/main" >> /etc/apk/reposit libxml2-dev \ libxslt-dev \ python \ - build-base" \ + build-base \ + protobuf-dev" \ && apk -U upgrade && apk add \ $BUILD_DEPS \ nodejs@edge \ @@ -29,6 +30,7 @@ RUN echo "@edge https://nl.alpinelinux.org/alpine/edge/main" >> /etc/apk/reposit file \ imagemagick@edge \ ca-certificates \ + protobuf \ && npm install -g npm@3 && npm install -g yarn \ && update-ca-certificates \ && rm -rf /tmp/* /var/cache/apk/* diff --git a/Gemfile b/Gemfile index 3651ad8b0..9c22b3c42 100644 --- a/Gemfile +++ b/Gemfile @@ -19,7 +19,7 @@ gem 'paperclip', '~> 5.1' gem 'paperclip-av-transcoder' gem 'addressable' -gem 'cld2', require: 'cld' +gem 'cld3', '~> 3.1.0' gem 'devise' gem 'devise-two-factor' gem 'doorkeeper' diff --git a/Gemfile.lock b/Gemfile.lock index 1baf6bec5..059fbab0e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -99,8 +99,8 @@ GEM rack-test (>= 0.5.4) xpath (~> 2.0) chunky_png (1.3.8) - cld2 (1.0.3) - ffi (~> 1.9.3) + cld3 (3.1.0) + ffi (>= 1.1.0, < 1.10.0) climate_control (0.1.0) cocaine (0.5.8) climate_control (>= 0.0.3, < 1.0) @@ -480,7 +480,7 @@ DEPENDENCIES capistrano-rbenv capistrano-yarn capybara - cld2 + cld3 (~> 3.1.0) devise devise-two-factor doorkeeper diff --git a/Vagrantfile b/Vagrantfile index b2ebbb83f..f2302b24f 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -33,7 +33,9 @@ sudo apt-get install \ redis-tools \ postgresql \ postgresql-contrib \ + protobuf-compiler \ yarn \ + libprotobuf-dev \ libreadline-dev \ -y diff --git a/app/lib/language_detector.rb b/app/lib/language_detector.rb index ca5cb2591..1c22a9ccc 100644 --- a/app/lib/language_detector.rb +++ b/app/lib/language_detector.rb @@ -6,6 +6,7 @@ class LanguageDetector def initialize(text, account = nil) @text = text @account = account + @identifier = CLD3::NNetLanguageIdentifier.new(1, 2048) end def to_iso_s @@ -15,15 +16,15 @@ class LanguageDetector private def detected_language_code - detected_language[:code].to_sym if detected_language_reliable? + result.language.to_sym if detected_language_reliable? end - def detected_language - @_detected_language ||= CLD.detect_language(text_without_urls) + def result + @result ||= @identifier.find_language(text_without_urls) end def detected_language_reliable? - detected_language[:reliable] + result.reliable? end def text_without_urls diff --git a/spec/lib/language_detector_spec.rb b/spec/lib/language_detector_spec.rb index 18ab5aee6..e543edd49 100644 --- a/spec/lib/language_detector_spec.rb +++ b/spec/lib/language_detector_spec.rb @@ -24,15 +24,15 @@ describe LanguageDetector do end describe 'when language can\'t be detected' do - it 'confirm language engine cant detect' do - result = CLD.detect_language('') - expect(result[:reliable]).to be false + it 'uses default locale when sent an empty document' do + result = described_class.new('').to_iso_s + expect(result).to eq :en end describe 'because of a URL' do it 'uses default locale when sent just a URL' do string = 'http://example.com/media/2kFTgOJLXhQf0g2nKB4' - cld_result = CLD.detect_language(string)[:code] + cld_result = CLD3::NNetLanguageIdentifier.new(0, 2048).find_language(string) expect(cld_result).not_to eq :en result = described_class.new(string).to_iso_s |