Add preview support for Microsoft Office and LibreOffice Writer files via Pandoc Markdown conversion (#8959).

Patch by Go MAEDA (user:maeda).


git-svn-id: https://svn.redmine.org/redmine/trunk@24485 e93f8b46-1217-0410-a6f0-8f06a7374b81

app/controllers/admin_controller.rb

@@ -79,7 +79,8 @@ class AdminController < ApplicationController
       [:text_all_migrations_have_been_run, !ActiveRecord::Base.connection.pool.migration_context.needs_migration?],
       [:text_minimagick_available,     Object.const_defined?(:MiniMagick)],
       [:text_convert_available,        Redmine::Thumbnail.convert_available?],
-      [:text_gs_available,             Redmine::Thumbnail.gs_available?]
+      [:text_gs_available,             Redmine::Thumbnail.gs_available?],
+      [:text_pandoc_available, Redmine::Markdownizer.available?]
     ]
     @checklist << [:text_default_active_job_queue_changed, Rails.application.config.active_job.queue_adapter != :async] if Rails.env.production?
   end

app/controllers/attachments_controller.rb

@@ -63,6 +63,8 @@ class AttachmentsController < ApplicationController
           render :action => 'image'
         elsif @attachment.is_pdf?
           render :action => 'pdf'
+        elsif @content = @attachment.markdownized_preview_content
+          render :action => 'markdownized'
         else
           render :action => 'other'
         end

app/models/attachment.rb

@@ -83,6 +83,13 @@ class Attachment < ApplicationRecord
   cattr_accessor :thumbnails_storage_path
   @@thumbnails_storage_path = File.join(Rails.root, "tmp", "thumbnails")
 
+  # Since markdownized previews can contain sensitive data, they should be
+  # stored under storage_path, which is expected to have appropriately
+  # restrictive permissions.
+  def self.markdownized_previews_storage_path
+    File.join(storage_path, 'markdownized_previews')
+  end
+
   before_create :files_to_final_location
   after_commit :delete_from_disk, :on => :destroy
   after_commit :reuse_existing_file_if_possible, :on => :create
@@ -267,6 +274,12 @@ class Attachment < ApplicationRecord
     end
   end
 
+  def self.clear_markdownized_previews
+    Dir.glob(File.join(markdownized_previews_storage_path, "*.md")).each do |file|
+      File.delete file
+    end
+  end
+
   def is_text?
     Redmine::MimeType.is_type?('text', filename) || Redmine::SyntaxHighlighting.filename_supported?(filename)
   end
@@ -299,6 +312,31 @@ class Attachment < ApplicationRecord
     Redmine::MimeType.is_type?('audio', filename)
   end
 
+  def markdownized_previewable?
+    readable? && Redmine::Markdownizer.available? && Redmine::Markdownizer.supports?(filename)
+  end
+
+  def markdownized_preview_content
+    return nil unless markdownized_previewable?
+
+    target = markdownized_preview_cache_path
+    if Redmine::Markdownizer.convert(diskfile, target)
+      File.read(target, :mode => "rb")
+    end
+  rescue => e
+    if logger
+      logger.error(
+        "An error occured while generating markdownized preview for #{disk_filename} " \
+          "to #{target}\nException was: #{e.message}"
+      )
+    end
+    nil
+  end
+
+  def markdownized_preview_cache_path
+    File.join(self.class.markdownized_previews_storage_path, "#{digest}_#{filesize}.md")
+  end
+
   def previewable?
     is_text? || is_image? || is_video? || is_audio?
   end
@@ -530,6 +568,7 @@ class Attachment < ApplicationRecord
     Dir[thumbnail_path("*")].each do |thumb|
       File.delete(thumb)
     end
+    FileUtils.rm_f(markdownized_preview_cache_path)
   end
 
   def thumbnail_path(size)

app/views/attachments/markdownized.html.erb

@@ -0,0 +1,4 @@
+<%= render :layout => 'layouts/file' do %>
+  <%= render :partial => 'common/markup',
+             :locals => {:markup_text_formatting => 'common_mark', :markup_text => @content} %>
+<% end %>

config/configuration.yml.example

@@ -237,6 +237,12 @@ default:
   #   - example.org
   #   - "*.example.com"
 
+  # Preview for Office documents (Microsoft Office / LibreOffice)
+  #
+  # Absolute path (e.g. /usr/local/bin/pandoc) to the Pandoc command
+  # used to convert supported attachments to Markdown for preview.
+  #pandoc_command:
+
 # specific configuration options for production environment
 # that overrides the default ones
 production:

config/locales/en.yml

@@ -1335,6 +1335,7 @@ en:
   text_minimagick_available: MiniMagick available (optional)
   text_convert_available: ImageMagick convert available (optional)
   text_gs_available: ImageMagick PDF support available (optional)
+  text_pandoc_available: Pandoc available (optional)
   text_default_active_job_queue_changed: Default queue adapter which is well suited only for dev/test changed
   text_destroy_time_entries_question: "%{hours} hours were reported on the issues you are about to delete. What do you want to do?"
   text_destroy_time_entries: Delete reported hours

config/locales/ja.yml

@@ -1103,6 +1103,7 @@ ja:
   notice_new_password_must_be_different: 新しいパスワードは現在のパスワードと異なるものでなければなりません
   setting_mail_handler_excluded_filenames: 除外する添付ファイル名
   text_convert_available: ImageMagickのconvertコマンドが利用可能 (オプション)
+  text_pandoc_available: Pandocが利用可能 (オプション)
   label_link: リンク
   label_only: 次のもののみ
   label_drop_down_list: ドロップダウンリスト

doc/INSTALL

@@ -19,6 +19,9 @@ Optional:
 * SCM binaries (e.g. svn, git...), for repository browsing (must be
   available in PATH)
 * ImageMagick (to enable Gantt export to png images)
+* Pandoc, to enable previews for Microsoft Office (.docx, .xlsx, .pptx) and
+  LibreOffice Writer (.odt) documents. Version 3.8.3 or later is recommended.
+  Older versions of Pandoc supports only .docx and .odt documents.
 
 Supported browsers:
 The current version of Firefox, Safari, Chrome, Chromium and Microsoft Edge.

lib/redmine/markdownizer.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+# Redmine - project management software
+# Copyright (C) 2006-  Jean-Philippe Lang
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+require 'fileutils'
+require 'shellwords'
+require 'tempfile'
+require 'timeout'
+
+module Redmine
+  module Markdownizer
+    extend Redmine::Utils::Shell
+
+    COMMAND = (Redmine::Configuration['pandoc_command'] || 'pandoc').freeze
+    MAX_SOURCE_SIZE = 20.megabytes
+    MAX_PREVIEW_SIZE = 512.kilobytes
+
+    def self.supports?(filename)
+      markdownizable_extensions.include?(File.extname(filename.to_s).downcase)
+    end
+
+    def self.convert(source, target)
+      return nil unless available?
+      return target if File.exist?(target)
+
+      if File.size(source) > MAX_SOURCE_SIZE
+        logger.warn("Markdownized preview generation skipped because source file is too large (#{File.size(source)} bytes): #{source}")
+        return nil
+      end
+
+      directory = File.dirname(target)
+      FileUtils.mkdir_p(directory)
+      args = [COMMAND, source, "-t", "gfm"]
+      pid = nil
+      output = Tempfile.new('markdownized-preview')
+
+      begin
+        Timeout.timeout(Redmine::Configuration['thumbnails_generation_timeout'].to_i) do
+          pid = Process.spawn(*args, out: output.path)
+          _, status = Process.wait2(pid)
+          unless status.success?
+            logger.error("Markdownized preview generation failed (#{status.exitstatus}):\nCommand: #{args.shelljoin}")
+            return nil
+          end
+        end
+      rescue Timeout::Error
+        if pid
+          Process.kill('KILL', pid)
+          Process.detach(pid)
+        end
+        logger.error("Markdownized preview generation timed out:\nCommand: #{args.shelljoin}")
+        return nil
+      rescue => e
+        logger.error("Markdownized preview generation failed:\nCommand: #{args.shelljoin}\nException was: #{e.message}")
+        return nil
+      ensure
+        output.close
+      end
+
+      preview = File.binread(output.path, MAX_PREVIEW_SIZE + 1) || +""
+      File.binwrite(target, preview.byteslice(0, MAX_PREVIEW_SIZE))
+      target
+    ensure
+      output&.unlink
+    end
+
+    def self.available?
+      return @available if defined?(@available)
+
+      begin
+        @pandoc_version = `#{shell_quote COMMAND} --version`[/pandoc\s+([\d.]+)/, 1].split('.').map(&:to_i)
+        @available = $?.success?
+      rescue
+        @available = false
+      end
+      logger.warn("Pandoc binary (#{COMMAND}) not available") unless @available
+      @available
+    end
+
+    def self.markdownizable_extensions
+      return @markdownizable_extensions if defined?(@markdownizable_extensions)
+
+      if available?
+        # Microsoft Word and LibreOffice Writer files are supported by a wide
+        # range of Pandoc versions
+        @markdownizable_extensions = %w[.docx .odt]
+      else
+        return (@markdownizable_extensions = [])
+      end
+      # Pandoc >= 3.8.3 supports Microsoft Excel and PowerPoint files
+      @markdownizable_extensions += %w[.xlsx .pptx] if (@pandoc_version <=> [3, 8, 3]) >= 0
+
+      @markdownizable_extensions
+    end
+
+    def self.logger
+      Rails.logger
+    end
+  end
+end

test/functional/attachments_controller_test.rb

@@ -23,9 +23,11 @@ class AttachmentsControllerTest < Redmine::ControllerTest
   def setup
     User.current = nil
     set_fixtures_attachments_directory
+    Attachment.clear_markdownized_previews
   end
 
   def teardown
+    Attachment.clear_markdownized_previews
     set_tmp_attachments_directory
   end
 
@@ -250,6 +252,50 @@ class AttachmentsControllerTest < Redmine::ControllerTest
     assert_select '.nodata', :text => 'No preview available'
   end
 
+  def test_show_msword
+    skip unless Redmine::Markdownizer.available?
+
+    set_tmp_attachments_directory
+    a = Attachment.new(
+      :container => Issue.find(1),
+      :file => uploaded_test_file(
+        'msword.docx',
+        'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
+      ),
+      :author => User.find(1)
+    )
+    assert a.save
+
+    get(:show, :params => {:id => a.id})
+
+    assert_response :success
+    assert_equal 'text/html', @response.media_type
+    assert_select 'div.filecontent.wiki', :text => /Redmine is a flexible project management web application/
+    assert_select '.nodata', :count => 0
+  end
+
+  def test_show_libreoffice_writer
+    skip unless Redmine::Markdownizer.available?
+
+    set_tmp_attachments_directory
+    a = Attachment.new(
+      :container => Issue.find(1),
+      :file => uploaded_test_file(
+        'libreoffice-writer.odt',
+        'application/vnd.oasis.opendocument.text'
+      ),
+      :author => User.find(1)
+    )
+    assert a.save
+
+    get(:show, :params => {:id => a.id})
+
+    assert_response :success
+    assert_equal 'text/html', @response.media_type
+    assert_select 'div.filecontent.wiki', :text => /Redmine is a flexible project management web application/
+    assert_select '.nodata', :count => 0
+  end
+
   def test_show_other_with_no_preview
     @request.session[:user_id] = 2
     get(:show, :params => {:id => 6})

test/unit/attachment_test.rb

@@ -23,6 +23,11 @@ class AttachmentTest < ActiveSupport::TestCase
   def setup
     User.current = nil
     set_tmp_attachments_directory
+    Attachment.clear_markdownized_previews
+  end
+
+  def teardown
+    Attachment.clear_markdownized_previews
   end
 
   def test_container_for_new_attachment_should_be_nil
@@ -528,6 +533,66 @@ class AttachmentTest < ActiveSupport::TestCase
     assert_equal false, Attachment.new(:filename => 'test.txt').thumbnailable?
   end
 
+  def test_markdownized_previewable_should_be_true_for_supported_extensions
+    skip unless Redmine::Markdownizer.available?
+
+    attachment = Attachment.new(
+      :container => Issue.find(1),
+      :file => uploaded_test_file(
+        "msword.docx",
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+      ),
+      :author => User.find(1)
+    )
+    assert attachment.save
+    assert_equal true, attachment.markdownized_previewable?
+  end
+
+  def test_markdownized_previewable_should_be_true_for_supported_libreoffice_extensions
+    skip unless Redmine::Markdownizer.available?
+
+    attachment = Attachment.new(
+      :container => Issue.find(1),
+      :file => uploaded_test_file(
+        "libreoffice-writer.odt",
+        "application/vnd.oasis.opendocument.text"
+      ),
+      :author => User.find(1)
+    )
+    assert attachment.save
+    assert_equal true, attachment.markdownized_previewable?
+  end
+
+  def test_markdownized_previewable_should_be_false_for_non_supported_extensions
+    skip unless Redmine::Markdownizer.available?
+
+    attachment = Attachment.new(
+      :container => Issue.find(1),
+      :file => uploaded_test_file("testfile.txt", "text/plain"),
+      :author => User.find(1)
+    )
+    assert attachment.save
+    assert_equal false, attachment.markdownized_previewable?
+  end
+
+  def test_delete_from_disk_should_delete_markdownized_preview_cache
+    attachment = Attachment.create!(
+      :container => Issue.find(1),
+      :file => uploaded_test_file(
+        "msword.docx",
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+      ),
+      :author => User.find(1)
+    )
+    preview = attachment.markdownized_preview_cache_path
+    FileUtils.mkdir_p(File.dirname(preview))
+    File.write(preview, "preview")
+    assert File.exist?(preview)
+
+    attachment.send(:delete_from_disk!)
+    assert_not File.exist?(preview)
+  end
+
   if convert_installed?
     def test_thumbnail_should_generate_the_thumbnail
       set_fixtures_attachments_directory