diff --git a/app/controllers/admin_controller.rb b/app/controllers/admin_controller.rb index 9b45f9553..59e519edf 100644 --- a/app/controllers/admin_controller.rb +++ b/app/controllers/admin_controller.rb @@ -79,7 +79,8 @@ class AdminController < ApplicationController [:text_all_migrations_have_been_run, !ActiveRecord::Base.connection.pool.migration_context.needs_migration?], [:text_minimagick_available, Object.const_defined?(:MiniMagick)], [:text_convert_available, Redmine::Thumbnail.convert_available?], - [:text_gs_available, Redmine::Thumbnail.gs_available?] + [:text_gs_available, Redmine::Thumbnail.gs_available?], + [:text_pandoc_available, Redmine::Markdownizer.available?] ] @checklist << [:text_default_active_job_queue_changed, Rails.application.config.active_job.queue_adapter != :async] if Rails.env.production? end diff --git a/app/controllers/attachments_controller.rb b/app/controllers/attachments_controller.rb index 18795891d..c4c9e0c16 100644 --- a/app/controllers/attachments_controller.rb +++ b/app/controllers/attachments_controller.rb @@ -63,6 +63,8 @@ class AttachmentsController < ApplicationController render :action => 'image' elsif @attachment.is_pdf? render :action => 'pdf' + elsif @content = @attachment.markdownized_preview_content + render :action => 'markdownized' else render :action => 'other' end diff --git a/app/models/attachment.rb b/app/models/attachment.rb index d0dd3a71f..12d4c809c 100644 --- a/app/models/attachment.rb +++ b/app/models/attachment.rb @@ -83,6 +83,13 @@ class Attachment < ApplicationRecord cattr_accessor :thumbnails_storage_path @@thumbnails_storage_path = File.join(Rails.root, "tmp", "thumbnails") + # Since markdownized previews can contain sensitive data, they should be + # stored under storage_path, which is expected to have appropriately + # restrictive permissions. + def self.markdownized_previews_storage_path + File.join(storage_path, 'markdownized_previews') + end + before_create :files_to_final_location after_commit :delete_from_disk, :on => :destroy after_commit :reuse_existing_file_if_possible, :on => :create @@ -267,6 +274,12 @@ class Attachment < ApplicationRecord end end + def self.clear_markdownized_previews + Dir.glob(File.join(markdownized_previews_storage_path, "*.md")).each do |file| + File.delete file + end + end + def is_text? Redmine::MimeType.is_type?('text', filename) || Redmine::SyntaxHighlighting.filename_supported?(filename) end @@ -299,6 +312,31 @@ class Attachment < ApplicationRecord Redmine::MimeType.is_type?('audio', filename) end + def markdownized_previewable? + readable? && Redmine::Markdownizer.available? && Redmine::Markdownizer.supports?(filename) + end + + def markdownized_preview_content + return nil unless markdownized_previewable? + + target = markdownized_preview_cache_path + if Redmine::Markdownizer.convert(diskfile, target) + File.read(target, :mode => "rb") + end + rescue => e + if logger + logger.error( + "An error occured while generating markdownized preview for #{disk_filename} " \ + "to #{target}\nException was: #{e.message}" + ) + end + nil + end + + def markdownized_preview_cache_path + File.join(self.class.markdownized_previews_storage_path, "#{digest}_#{filesize}.md") + end + def previewable? is_text? || is_image? || is_video? || is_audio? end @@ -530,6 +568,7 @@ class Attachment < ApplicationRecord Dir[thumbnail_path("*")].each do |thumb| File.delete(thumb) end + FileUtils.rm_f(markdownized_preview_cache_path) end def thumbnail_path(size) diff --git a/app/views/attachments/markdownized.html.erb b/app/views/attachments/markdownized.html.erb new file mode 100644 index 000000000..3a158e827 --- /dev/null +++ b/app/views/attachments/markdownized.html.erb @@ -0,0 +1,4 @@ +<%= render :layout => 'layouts/file' do %> + <%= render :partial => 'common/markup', + :locals => {:markup_text_formatting => 'common_mark', :markup_text => @content} %> +<% end %> diff --git a/config/configuration.yml.example b/config/configuration.yml.example index e37f38745..cdfd516b5 100644 --- a/config/configuration.yml.example +++ b/config/configuration.yml.example @@ -237,6 +237,12 @@ default: # - example.org # - "*.example.com" + # Preview for Office documents (Microsoft Office / LibreOffice) + # + # Absolute path (e.g. /usr/local/bin/pandoc) to the Pandoc command + # used to convert supported attachments to Markdown for preview. + #pandoc_command: + # specific configuration options for production environment # that overrides the default ones production: diff --git a/config/locales/en.yml b/config/locales/en.yml index 17dfdfb6d..1ea7b4170 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -1335,6 +1335,7 @@ en: text_minimagick_available: MiniMagick available (optional) text_convert_available: ImageMagick convert available (optional) text_gs_available: ImageMagick PDF support available (optional) + text_pandoc_available: Pandoc available (optional) text_default_active_job_queue_changed: Default queue adapter which is well suited only for dev/test changed text_destroy_time_entries_question: "%{hours} hours were reported on the issues you are about to delete. What do you want to do?" text_destroy_time_entries: Delete reported hours diff --git a/config/locales/ja.yml b/config/locales/ja.yml index 53bbc362f..6749d58e5 100644 --- a/config/locales/ja.yml +++ b/config/locales/ja.yml @@ -1103,6 +1103,7 @@ ja: notice_new_password_must_be_different: 新しいパスワードは現在のパスワードと異なるものでなければなりません setting_mail_handler_excluded_filenames: 除外する添付ファイル名 text_convert_available: ImageMagickのconvertコマンドが利用可能 (オプション) + text_pandoc_available: Pandocが利用可能 (オプション) label_link: リンク label_only: 次のもののみ label_drop_down_list: ドロップダウンリスト diff --git a/doc/INSTALL b/doc/INSTALL index 6d70f683a..aa406cfa3 100644 --- a/doc/INSTALL +++ b/doc/INSTALL @@ -19,6 +19,9 @@ Optional: * SCM binaries (e.g. svn, git...), for repository browsing (must be available in PATH) * ImageMagick (to enable Gantt export to png images) +* Pandoc, to enable previews for Microsoft Office (.docx, .xlsx, .pptx) and + LibreOffice Writer (.odt) documents. Version 3.8.3 or later is recommended. + Older versions of Pandoc supports only .docx and .odt documents. Supported browsers: The current version of Firefox, Safari, Chrome, Chromium and Microsoft Edge. diff --git a/lib/redmine/markdownizer.rb b/lib/redmine/markdownizer.rb new file mode 100644 index 000000000..688a2afaf --- /dev/null +++ b/lib/redmine/markdownizer.rb @@ -0,0 +1,115 @@ +# frozen_string_literal: true + +# Redmine - project management software +# Copyright (C) 2006- Jean-Philippe Lang +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +require 'fileutils' +require 'shellwords' +require 'tempfile' +require 'timeout' + +module Redmine + module Markdownizer + extend Redmine::Utils::Shell + + COMMAND = (Redmine::Configuration['pandoc_command'] || 'pandoc').freeze + MAX_SOURCE_SIZE = 20.megabytes + MAX_PREVIEW_SIZE = 512.kilobytes + + def self.supports?(filename) + markdownizable_extensions.include?(File.extname(filename.to_s).downcase) + end + + def self.convert(source, target) + return nil unless available? + return target if File.exist?(target) + + if File.size(source) > MAX_SOURCE_SIZE + logger.warn("Markdownized preview generation skipped because source file is too large (#{File.size(source)} bytes): #{source}") + return nil + end + + directory = File.dirname(target) + FileUtils.mkdir_p(directory) + args = [COMMAND, source, "-t", "gfm"] + pid = nil + output = Tempfile.new('markdownized-preview') + + begin + Timeout.timeout(Redmine::Configuration['thumbnails_generation_timeout'].to_i) do + pid = Process.spawn(*args, out: output.path) + _, status = Process.wait2(pid) + unless status.success? + logger.error("Markdownized preview generation failed (#{status.exitstatus}):\nCommand: #{args.shelljoin}") + return nil + end + end + rescue Timeout::Error + if pid + Process.kill('KILL', pid) + Process.detach(pid) + end + logger.error("Markdownized preview generation timed out:\nCommand: #{args.shelljoin}") + return nil + rescue => e + logger.error("Markdownized preview generation failed:\nCommand: #{args.shelljoin}\nException was: #{e.message}") + return nil + ensure + output.close + end + + preview = File.binread(output.path, MAX_PREVIEW_SIZE + 1) || +"" + File.binwrite(target, preview.byteslice(0, MAX_PREVIEW_SIZE)) + target + ensure + output&.unlink + end + + def self.available? + return @available if defined?(@available) + + begin + @pandoc_version = `#{shell_quote COMMAND} --version`[/pandoc\s+([\d.]+)/, 1].split('.').map(&:to_i) + @available = $?.success? + rescue + @available = false + end + logger.warn("Pandoc binary (#{COMMAND}) not available") unless @available + @available + end + + def self.markdownizable_extensions + return @markdownizable_extensions if defined?(@markdownizable_extensions) + + if available? + # Microsoft Word and LibreOffice Writer files are supported by a wide + # range of Pandoc versions + @markdownizable_extensions = %w[.docx .odt] + else + return (@markdownizable_extensions = []) + end + # Pandoc >= 3.8.3 supports Microsoft Excel and PowerPoint files + @markdownizable_extensions += %w[.xlsx .pptx] if (@pandoc_version <=> [3, 8, 3]) >= 0 + + @markdownizable_extensions + end + + def self.logger + Rails.logger + end + end +end diff --git a/test/fixtures/files/libreoffice-writer.odt b/test/fixtures/files/libreoffice-writer.odt new file mode 100644 index 000000000..32ab500ff Binary files /dev/null and b/test/fixtures/files/libreoffice-writer.odt differ diff --git a/test/fixtures/files/msword.docx b/test/fixtures/files/msword.docx new file mode 100644 index 000000000..2b438db96 Binary files /dev/null and b/test/fixtures/files/msword.docx differ diff --git a/test/functional/attachments_controller_test.rb b/test/functional/attachments_controller_test.rb index bc664cea5..623805202 100644 --- a/test/functional/attachments_controller_test.rb +++ b/test/functional/attachments_controller_test.rb @@ -23,9 +23,11 @@ class AttachmentsControllerTest < Redmine::ControllerTest def setup User.current = nil set_fixtures_attachments_directory + Attachment.clear_markdownized_previews end def teardown + Attachment.clear_markdownized_previews set_tmp_attachments_directory end @@ -250,6 +252,50 @@ class AttachmentsControllerTest < Redmine::ControllerTest assert_select '.nodata', :text => 'No preview available' end + def test_show_msword + skip unless Redmine::Markdownizer.available? + + set_tmp_attachments_directory + a = Attachment.new( + :container => Issue.find(1), + :file => uploaded_test_file( + 'msword.docx', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + ), + :author => User.find(1) + ) + assert a.save + + get(:show, :params => {:id => a.id}) + + assert_response :success + assert_equal 'text/html', @response.media_type + assert_select 'div.filecontent.wiki', :text => /Redmine is a flexible project management web application/ + assert_select '.nodata', :count => 0 + end + + def test_show_libreoffice_writer + skip unless Redmine::Markdownizer.available? + + set_tmp_attachments_directory + a = Attachment.new( + :container => Issue.find(1), + :file => uploaded_test_file( + 'libreoffice-writer.odt', + 'application/vnd.oasis.opendocument.text' + ), + :author => User.find(1) + ) + assert a.save + + get(:show, :params => {:id => a.id}) + + assert_response :success + assert_equal 'text/html', @response.media_type + assert_select 'div.filecontent.wiki', :text => /Redmine is a flexible project management web application/ + assert_select '.nodata', :count => 0 + end + def test_show_other_with_no_preview @request.session[:user_id] = 2 get(:show, :params => {:id => 6}) diff --git a/test/unit/attachment_test.rb b/test/unit/attachment_test.rb index 7bfa9311f..1af8eb0f2 100644 --- a/test/unit/attachment_test.rb +++ b/test/unit/attachment_test.rb @@ -23,6 +23,11 @@ class AttachmentTest < ActiveSupport::TestCase def setup User.current = nil set_tmp_attachments_directory + Attachment.clear_markdownized_previews + end + + def teardown + Attachment.clear_markdownized_previews end def test_container_for_new_attachment_should_be_nil @@ -528,6 +533,66 @@ class AttachmentTest < ActiveSupport::TestCase assert_equal false, Attachment.new(:filename => 'test.txt').thumbnailable? end + def test_markdownized_previewable_should_be_true_for_supported_extensions + skip unless Redmine::Markdownizer.available? + + attachment = Attachment.new( + :container => Issue.find(1), + :file => uploaded_test_file( + "msword.docx", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + ), + :author => User.find(1) + ) + assert attachment.save + assert_equal true, attachment.markdownized_previewable? + end + + def test_markdownized_previewable_should_be_true_for_supported_libreoffice_extensions + skip unless Redmine::Markdownizer.available? + + attachment = Attachment.new( + :container => Issue.find(1), + :file => uploaded_test_file( + "libreoffice-writer.odt", + "application/vnd.oasis.opendocument.text" + ), + :author => User.find(1) + ) + assert attachment.save + assert_equal true, attachment.markdownized_previewable? + end + + def test_markdownized_previewable_should_be_false_for_non_supported_extensions + skip unless Redmine::Markdownizer.available? + + attachment = Attachment.new( + :container => Issue.find(1), + :file => uploaded_test_file("testfile.txt", "text/plain"), + :author => User.find(1) + ) + assert attachment.save + assert_equal false, attachment.markdownized_previewable? + end + + def test_delete_from_disk_should_delete_markdownized_preview_cache + attachment = Attachment.create!( + :container => Issue.find(1), + :file => uploaded_test_file( + "msword.docx", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + ), + :author => User.find(1) + ) + preview = attachment.markdownized_preview_cache_path + FileUtils.mkdir_p(File.dirname(preview)) + File.write(preview, "preview") + assert File.exist?(preview) + + attachment.send(:delete_from_disk!) + assert_not File.exist?(preview) + end + if convert_installed? def test_thumbnail_should_generate_the_thumbnail set_fixtures_attachments_directory