From 1633a0b0ed3744a1ccb44913accbdc1fe099bdad Mon Sep 17 00:00:00 2001 From: Luc Didry Date: Fri, 17 Apr 2020 18:49:10 +0200 Subject: [PATCH] WIP - Swift storage --- cpanfile | 3 + cpanfile.snapshot | 239 ++++++++++++++++++++++-- lib/Lutim.pm | 5 + lib/Lutim/Command/cron/cleanfiles.pm | 4 +- lib/Lutim/Command/cron/watch.pm | 2 +- lib/Lutim/Command/image.pm | 2 +- lib/Lutim/Controller/Image.pm | 47 ++--- lib/Lutim/DB/Image.pm | 166 +++++++++++++++- lib/Lutim/Plugin/Helpers.pm | 109 +++++------ lutim.conf.template | 12 ++ themes/default/lib/Lutim/I18N/en.po | 18 +- themes/default/lib/Lutim/I18N/lutim.pot | 18 +- 12 files changed, 511 insertions(+), 114 deletions(-) diff --git a/cpanfile b/cpanfile index 8cd5af3..cb345d6 100644 --- a/cpanfile +++ b/cpanfile @@ -53,6 +53,9 @@ feature 'htpasswd', 'Htpasswd authentication support' => sub { requires 'Apache::Htpasswd'; requires 'Mojolicious::Plugin::Authentication'; }; +feature 'swift-storage', 'Openstack Swift object storage support' => sub { + requires 'Net::OpenStack::Swift'; +}; feature 'test' => sub { requires 'Devel::Cover'; }; diff --git a/cpanfile.snapshot b/cpanfile.snapshot index eb11ab2..abfbd31 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -9,6 +9,23 @@ DISTRIBUTIONS Digest::SHA 2 ExtUtils::MakeMaker 0 MIME::Base64 0 + App-Rad-1.05 + pathname: G/GA/GARU/App-Rad-1.05.tar.gz + provides: + App::Rad 1.05 + App::Rad::Config undef + App::Rad::Exclude 0.01 + App::Rad::Help 0.03 + App::Rad::Include 0.01 + requirements: + Attribute::Handlers 0 + B::Deparse 0 + Carp 0 + ExtUtils::MakeMaker 0 + File::Temp 0 + FindBin 0 + Getopt::Long 2.36 + Test::More 0 Archive-Zip-1.60 pathname: P/PH/PHRED/Archive-Zip-1.60.tar.gz provides: @@ -173,6 +190,12 @@ DISTRIBUTIONS strict 0 vars 0 warnings 0 + Class-Accessor-Lite-0.08 + pathname: K/KA/KAZUHO/Class-Accessor-Lite-0.08.tar.gz + provides: + Class::Accessor::Lite 0.08 + requirements: + ExtUtils::MakeMaker 6.36 Class-Data-Inheritable-0.08 pathname: T/TM/TMTM/Class-Data-Inheritable-0.08.tar.gz provides: @@ -537,6 +560,23 @@ DISTRIBUTIONS Data::Validate::Domain 0 Data::Validate::IP 0 ExtUtils::MakeMaker 0 + Data-Validator-1.07 + pathname: G/GF/GFUJI/Data-Validator-1.07.tar.gz + provides: + Data::Validator 1.07 + Data::Validator::Role::AllowExtra undef + Data::Validator::Role::Croak undef + Data::Validator::Role::Method undef + Data::Validator::Role::NoRestricted undef + Data::Validator::Role::NoThrow undef + Data::Validator::Role::Sequenced undef + Data::Validator::Role::SmartSequenced undef + Data::Validator::Role::StrictSequenced undef + requirements: + ExtUtils::MakeMaker 6.59 + Module::Build 0.38 + Mouse 0.93 + perl 5.008001 DateTime-1.48 pathname: D/DR/DROLSKY/DateTime-1.48.tar.gz provides: @@ -995,6 +1035,19 @@ DISTRIBUTIONS perl 5.008004 strict 0 warnings 0 + Devel-CheckCompiler-0.07 + pathname: S/SY/SYOHEX/Devel-CheckCompiler-0.07.tar.gz + provides: + Devel::AssertC99 undef + Devel::CheckCompiler 0.07 + requirements: + Exporter 0 + ExtUtils::CBuilder 0 + File::Temp 0 + Module::Build::Tiny 0.035 + Test::More 0.98 + parent 0 + perl 5.008001 Devel-Cover-1.29 pathname: P/PJ/PJCJ/Devel-Cover-1.29.tar.gz provides: @@ -1077,6 +1130,13 @@ DISTRIBUTIONS perl 5.006 strict 0 warnings 0 + Devel-PPPort-3.57 + pathname: A/AT/ATOOMIC/Devel-PPPort-3.57.tar.gz + provides: + Devel::PPPort 3.57 + requirements: + ExtUtils::MakeMaker 0 + FindBin 0 Devel-StackTrace-2.03 pathname: D/DR/DROLSKY/Devel-StackTrace-2.03.tar.gz provides: @@ -1329,6 +1389,27 @@ DISTRIBUTIONS Test::More 0 Test::Warn 0 perl 5.006 + Furl-3.13 + pathname: T/TO/TOKUHIROM/Furl-3.13.tar.gz + provides: + Furl 3.13 + Furl::ConnectionCache undef + Furl::HTTP 3.13 + Furl::Headers undef + Furl::Request undef + Furl::Response undef + Furl::ZlibStream undef + requirements: + Class::Accessor::Lite 0 + Encode 0 + HTTP::Parser::XS 0.11 + MIME::Base64 0 + Module::Build::Tiny 0.035 + Mozilla::CA 0 + Scalar::Util 0 + Socket 0 + Time::HiRes 0 + perl 5.008001 HTML-Parser-3.72 pathname: G/GA/GAAS/HTML-Parser-3.72.tar.gz provides: @@ -1447,6 +1528,14 @@ DISTRIBUTIONS ExtUtils::MakeMaker 0 HTTP::Headers 6 perl 5.008001 + HTTP-Parser-XS-0.17 + pathname: K/KA/KAZUHO/HTTP-Parser-XS-0.17.tar.gz + provides: + HTTP::Parser::XS 0.17 + HTTP::Parser::XS::PP undef + requirements: + ExtUtils::MakeMaker 6.36 + Test::More 0.96 Hash-Merge-0.300 pathname: R/RE/REHSACK/Hash-Merge-0.300.tar.gz provides: @@ -1802,6 +1891,18 @@ DISTRIBUTIONS constant 0 strict 0 warnings 0 + Log-Minimal-0.19 + pathname: K/KA/KAZEBURO/Log-Minimal-0.19.tar.gz + provides: + Log::Minimal 0.19 + requirements: + CPAN::Meta 0 + CPAN::Meta::Prereqs 0 + Data::Dumper 0 + ExtUtils::CBuilder 0 + Module::Build 0.38 + Scalar::Util 0 + Term::ANSIColor 0 MRO-Compat-0.13 pathname: H/HA/HAARG/MRO-Compat-0.13.tar.gz provides: @@ -1918,6 +2019,21 @@ DISTRIBUTIONS perl 5.006 strict 0 warnings 0 + Module-Build-XSUtil-0.19 + pathname: H/HI/HIDEAKIO/Module-Build-XSUtil-0.19.tar.gz + provides: + Module::Build::XSUtil 0.19 + requirements: + Devel::CheckCompiler 0 + Devel::PPPort 0 + Exporter 0 + ExtUtils::CBuilder 0 + File::Basename 0 + File::Path 0 + Module::Build 0.4005 + XSLoader 0 + parent 0 + perl 5.008001 Module-Implementation-0.09 pathname: D/DR/DROLSKY/Module-Implementation-0.09.tar.gz provides: @@ -2082,7 +2198,6 @@ DISTRIBUTIONS Mojolicious::Lite undef Mojolicious::Plugin undef Mojolicious::Plugin::Config undef - Mojolicious::Plugin::Config::Sandbox undef Mojolicious::Plugin::DefaultHelpers undef Mojolicious::Plugin::EPLRenderer undef Mojolicious::Plugin::EPRenderer undef @@ -2605,6 +2720,44 @@ DISTRIBUTIONS parent 0.223 strict 1.03 warnings 1.03 + Mouse-v2.5.9 + pathname: S/SK/SKAJI/Mouse-v2.5.9.tar.gz + provides: + Mouse v2.5.9 + Mouse::Exporter undef + Mouse::Meta::Attribute undef + Mouse::Meta::Class undef + Mouse::Meta::Method undef + Mouse::Meta::Method::Accessor undef + Mouse::Meta::Method::Constructor undef + Mouse::Meta::Method::Delegation undef + Mouse::Meta::Method::Destructor undef + Mouse::Meta::Module undef + Mouse::Meta::Role undef + Mouse::Meta::Role::Application undef + Mouse::Meta::Role::Application::RoleSummation undef + Mouse::Meta::Role::Composite undef + Mouse::Meta::Role::Method undef + Mouse::Meta::TypeConstraint undef + Mouse::Object undef + Mouse::PurePerl undef + Mouse::Role v2.5.9 + Mouse::Spec v2.5.9 + Mouse::TypeRegistry undef + Mouse::Util v2.5.9 + Mouse::Util::MetaRole undef + Mouse::Util::TypeConstraints undef + Squirrel undef + Squirrel::Role undef + Test::Mouse undef + ouse undef + requirements: + ExtUtils::CBuilder 0 + Module::Build 0.4005 + Module::Build::XSUtil 0.19 + Scalar::Util 1.14 + XSLoader 0.02 + perl 5.008005 Mozilla-CA-20180117 pathname: A/AB/ABH/Mozilla-CA-20180117.tar.gz provides: @@ -2640,6 +2793,31 @@ DISTRIBUTIONS strict 0 vars 0 warnings 0 + Net-OpenStack-Swift-0.15 + pathname: M/MA/MASAKYST/Net-OpenStack-Swift-0.15.tar.gz + provides: + Net::OpenStack::Swift 0.15 + Net::OpenStack::Swift::InnerKeystone::Base undef + Net::OpenStack::Swift::InnerKeystone::V1_0 undef + Net::OpenStack::Swift::InnerKeystone::V2_0 undef + Net::OpenStack::Swift::InnerKeystone::V3_0 undef + Net::OpenStack::Swift::Util undef + requirements: + App::Rad 0 + Data::Validator 0 + Furl 0 + IO::Socket::SSL 0 + JSON 0 + Log::Minimal 0 + Module::Build::Tiny 0.035 + Mouse 0 + Parallel::Fork::BossWorkerAsync 0 + Path::Tiny 0 + Sys::CPU 0 + Text::ASCIITable 0 + URI::Escape 0 + namespace::clean 0 + perl 5.010_001 Net-SSLeay-1.85 pathname: M/MI/MIKEM/Net-SSLeay-1.85.tar.gz provides: @@ -2707,6 +2885,12 @@ DISTRIBUTIONS XSLoader 0 strict 0 warnings 0 + Parallel-Fork-BossWorkerAsync-0.09 + pathname: J/JV/JVANNUCCI/Parallel-Fork-BossWorkerAsync-0.09.tar.gz + provides: + Parallel::Fork::BossWorkerAsync 0.09 + requirements: + ExtUtils::MakeMaker 0 Params-Classify-0.015 pathname: Z/ZE/ZEFRAM/Params-Classify-0.015.tar.gz provides: @@ -2749,6 +2933,32 @@ DISTRIBUTIONS overload 0 strict 0 warnings 0 + Path-Tiny-0.112 + pathname: D/DA/DAGOLDEN/Path-Tiny-0.112.tar.gz + provides: + Path::Tiny 0.112 + Path::Tiny::Error 0.112 + requirements: + Carp 0 + Cwd 0 + Digest 1.03 + Digest::SHA 5.45 + Encode 0 + Exporter 5.57 + ExtUtils::MakeMaker 6.17 + Fcntl 0 + File::Copy 0 + File::Glob 0 + File::Path 2.07 + File::Spec 0.86 + File::Temp 0.19 + File::stat 0 + constant 0 + overload 0 + perl 5.008001 + strict 0 + warnings 0 + warnings::register 0 Role-Tiny-2.000006 pathname: H/HA/HAARG/Role-Tiny-2.000006.tar.gz provides: @@ -2774,17 +2984,6 @@ DISTRIBUTIONS Sub::Quote 2.000001 Text::Balanced 2.00 perl 5.006 - Scalar-List-Utils-1.50 - pathname: P/PE/PEVANS/Scalar-List-Utils-1.50.tar.gz - provides: - List::Util 1.50 - List::Util::XS 1.50 - Scalar::Util 1.50 - Sub::Util 1.50 - requirements: - ExtUtils::MakeMaker 0 - Test::More 0 - perl 5.006 Specio-0.42 pathname: D/DR/DROLSKY/Specio-0.42.tar.gz provides: @@ -2948,6 +3147,12 @@ DISTRIBUTIONS Text::Balanced 2 if 0 perl 5.005 + Sys-CPU-0.52 + pathname: M/MK/MKODERER/Sys-CPU-0.52.tar.gz + provides: + Sys::CPU 0.52 + requirements: + ExtUtils::MakeMaker 0 Task-Weaken-1.06 pathname: E/ET/ETHER/Task-Weaken-1.06.tar.gz provides: @@ -2993,6 +3198,16 @@ DISTRIBUTIONS Test::Builder 0.13 Test::Builder::Tester 1.02 perl 5.006 + Text-ASCIITable-0.22 + pathname: L/LU/LUNATIC/Text-ASCIITable-0.22.tar.gz + provides: + Text::ASCIITable 0.22 + Text::ASCIITable::Wrap 0.2 + requirements: + Carp 0 + Encode 0 + List::Util 0 + perl v5.6.0 Text-Soundex-3.05 pathname: R/RJ/RJBS/Text-Soundex-3.05.tar.gz provides: diff --git a/lib/Lutim.pm b/lib/Lutim.pm index 9d8ca69..60dd3a2 100644 --- a/lib/Lutim.pm +++ b/lib/Lutim.pm @@ -109,6 +109,11 @@ sub startup { $self->plugin('Lutim::Plugin::Helpers'); $self->plugin('Lutim::Plugin::Lang'); + # Now helpers has been loaded, time to check Swift container + if ($config->{swift}) { + $self->check_swift_container(); + } + # Minion if ($config->{minion}->{enabled}) { $self->config->{minion}->{dbtype} = 'sqlite' unless defined $config->{minion}->{dbtype}; diff --git a/lib/Lutim/Command/cron/cleanfiles.pm b/lib/Lutim/Command/cron/cleanfiles.pm index 4d6f0f7..a12881e 100644 --- a/lib/Lutim/Command/cron/cleanfiles.pm +++ b/lib/Lutim/Command/cron/cleanfiles.pm @@ -33,7 +33,7 @@ sub run { $dbi->get_images_to_clean()->each( sub { my ($img, $num) = @_; - $l->app->delete_image($img); + $img->delete(); } ); @@ -42,7 +42,7 @@ sub run { $dbi->get_no_longer_viewed_files($time)->each( sub { my ($img, $num) = @_; - $l->app->delete_image($img); + $img->delete(); } ); } diff --git a/lib/Lutim/Command/cron/watch.pm b/lib/Lutim/Command/cron/watch.pm index 8b8b7e3..75f5719 100644 --- a/lib/Lutim/Command/cron/watch.pm +++ b/lib/Lutim/Command/cron/watch.pm @@ -50,7 +50,7 @@ sub run { $dbi->get_50_oldest()->each( sub { my ($img, $num) = @_; - $l->app->delete_image($img); + $img->delete(); } ); } while (du(qw/files/) > $config->{max_total_size}); diff --git a/lib/Lutim/Command/image.pm b/lib/Lutim/Command/image.pm index b4ad261..b35e1f0 100644 --- a/lib/Lutim/Command/image.pm +++ b/lib/Lutim/Command/image.pm @@ -173,7 +173,7 @@ sub delete_short { chomp $confirm; } if ($confirm =~ m/^y(es)?$/i) { - $c->app->delete_image($i); + $i->delete(); } else { say 'Answer was not "y" or "yes". Aborting deletion.'; } diff --git a/lib/Lutim/Controller/Image.pm b/lib/Lutim/Controller/Image.pm index 6441f9a..51c2ffa 100644 --- a/lib/Lutim/Controller/Image.pm +++ b/lib/Lutim/Controller/Image.pm @@ -231,7 +231,7 @@ sub delete { } else { $c->app->log->info('[DELETION] someone made '.$image->filename.' removed with token method (path: '.$image->path.')') unless $c->config('quiet_logs'); - $c->delete_image($image); + $image->delete(); return $c->respond_to( json => { json => { @@ -397,7 +397,8 @@ sub add { # Save file and create record my $filename = unidecode($upload->filename); my $ext = ($filename =~ m/([^.]+)$/)[0]; - my $path = 'files/'.$record->short.'.'.$ext; + my $path = $record->short.'.'.$ext; + $path = Mojo::File->new('files', $path)->to_string unless ($c->app->config('swift')); my ($width, $height); if ($im_loaded && $mediatype ne 'image/svg+xml' && $mediatype !~ m#image/(x-)?xcf# && $mediatype ne 'image/webp') { # ImageMagick don't work in Debian with svg (for now?) @@ -493,12 +494,11 @@ sub add { if ($c->param('crypt') || $c->config('always_encrypt')) { ($upload, $key, $iv) = $c->crypt($upload, $filename); } - $upload->move_to($path); $record->path($path) ->filename($filename) ->mediatype($mediatype) - ->footprint(digest_file_hex($path, 'SHA-512')) + ->footprint(digest_file_hex($upload->asset->to_file->path, 'SHA-512')) ->enabled(1) ->delete_at_day(($c->param('delete-day') && ($c->param('delete-day') <= $c->max_delay || $c->max_delay == 0)) ? $c->param('delete-day') : $c->max_delay) ->delete_at_first_view(($c->param('first-view'))? 1 : 0) @@ -507,7 +507,8 @@ sub add { ->width($width) ->height($height) ->iv($iv) - ->write; + ->write + ->store($upload); # Log image creation $c->app->log->info('[CREATION] '.$ip.' pushed '.$filename.' (path: '.$path.')') unless $c->config('quiet_logs'); @@ -605,12 +606,13 @@ sub short { my $image = Lutim::DB::Image->new(app => $c->app, short => $short); if ($image->enabled && $image->path) { + # Image deleted if($image->delete_at_day && $image->created_at + $image->delete_at_day * 86400 <= time()) { # Log deletion $c->app->log->info('[DELETION] someone tried to view '.$image->filename.' but it has been removed by expiration (path: '.$image->path.')') unless $c->config('quiet_logs'); # Delete image - $c->delete_image($image); + $image->delete(); # Warn user $c->flash( @@ -620,6 +622,7 @@ sub short { } my $test; + # Twitter page if (defined($touit) && $image->mediatype !~ m/svg/) { $test = 1; my $short = $image->short; @@ -629,11 +632,11 @@ sub short { if (defined($image->width) && defined($image->height)) { ($width, $height) = ($image->width, $image->height); } elsif ($im_loaded) { - my $upload = $c->decrypt($key, $image->path, $image->iv); - my $im = Image::Magick->new; - $im->BlobToImage($upload->slurp); - $width = $im->Get('width'); - $height = $im->Get('height'); + my $tmp = $image->decrypt($key); + my $im = Image::Magick->new; + $im->BlobToImage($tmp->slurp); + $width = $im->Get('width'); + $height = $im->Get('height'); $image->width($width) ->height($height) @@ -656,7 +659,7 @@ sub short { $c->app->log->info('[DELETION] someone made '.$image->filename.' removed (path: '.$image->path.')') unless $c->config('quiet_logs'); # Delete image - $c->delete_image($image); + $image->delete(); $c->flash( msg => $c->l('Unable to find the image: it has been deleted.') @@ -688,7 +691,7 @@ sub short { $c->app->log->info('[DELETION] someone made '.$image->filename.' removed (path: '.$image->path.')') unless $c->config('quiet_logs'); # Delete image - $c->delete_image($image); + $image->delete(); } }); } else { @@ -740,7 +743,7 @@ sub zip { $c->app->log->info('[DELETION] someone tried to view '.$image->filename.' but it has been removed by expiration (path: '.$image->path.')') unless $c->config('quiet_logs'); # Delete image - $c->delete_image($image); + $image->delete(); # Warn user $zip->addString(encode('UTF-8', $c->l('Unable to find the image: it has been deleted.')), 'images/'.$filename.'.txt'); @@ -753,7 +756,7 @@ sub zip { $c->app->log->info('[DELETION] someone made '.$image->filename.' removed (path: '.$image->path.')') unless $c->config('quiet_logs'); # Delete image - $c->delete_image($image); + $image->delete(); $zip->addString(encode('UTF-8', $c->l('Unable to find the image: it has been deleted.')), 'images/'.$filename.'.txt'); next; @@ -764,16 +767,18 @@ sub zip { $expires = $dt->strftime("%a, %d %b %Y %H:%M:%S GMT"); my $path = $image->path; - unless ( -f $path && -r $path ) { - $c->app->log->error("Cannot read file [$path]. error [$!]"); - $zip->addString(encode('UTF-8', $c->l('Unable to find the image: it has been deleted.')), 'images/'.$filename.'.txt'); - next; + unless ($c->app->config('swift')) { + unless ( -f $path && -r $path ) { + $c->app->log->error("Cannot read file [$path]. error [$!]"); + $zip->addString(encode('UTF-8', $c->l('Unable to find the image: it has been deleted.')), 'images/'.$filename.'.txt'); + next; + } } if ($key) { - $zip->addString($c->decrypt($key, $path, $image->iv), "images/$filename"); + $zip->addString($image->decrypt($key), "images/$filename"); } else { - $zip->addFile($path, "images/$filename"); + $zip->addString($image->retrieve, "images/$filename"); } # Log access diff --git a/lib/Lutim/DB/Image.pm b/lib/Lutim/DB/Image.pm index 9722ee8..6739928 100644 --- a/lib/Lutim/DB/Image.pm +++ b/lib/Lutim/DB/Image.pm @@ -1,6 +1,10 @@ # vim:set sw=4 ts=4 sts=4 ft=perl expandtab: package Lutim::DB::Image; +use bytes; use Mojo::Base -base; +use Mojo::File; +use Crypt::CBC; +use File::Temp qw(tempfile); has 'short'; has 'path'; @@ -20,6 +24,8 @@ has 'height'; has 'iv'; has 'app'; +=encoding utf8 + =head1 NAME Lutim::DB::Image - DB abstraction layer for Lutim images @@ -151,13 +157,11 @@ sub to_hash { =over 1 -=item B : C<$c-Ecount_delete_at_day_endis($delete_at_day, $enabled[, $time])> +=item B : C<$c-Ecount_delete_at_day_endis($delete_at_day, $enabled, [$time])> -=item B : two mandatory parameters: one integer, the delete_at_day attribute, a boolean (0 or 1), the enabled attribute - an optional parameter: an unix timestamp +=item B : two mandatory parameters: one integer, the delete_at_day attribute, a boolean (0 or 1), the enabled attribute and an optional parameter: an unix timestamp. -=item B : count how many images there are with the given delete_at_day attribute, and enabled or disabled, depending on the given enabled attribute - if the optional parameter is given, count only images according to the given mandatory parameters that were created before the timestamp +=item B : count how many images there are with the given delete_at_day attribute, and enabled or disabled, depending on the given enabled attribute. If the optional parameter is given, count only images according to the given mandatory parameters that were created before the timestamp =item B : integer @@ -345,6 +349,158 @@ sub to_hash { =back +=head2 store + +=over 1 + +=item B : C<$c-Estore($upload)> + +=item B : a Mojo::Upload object + +=item B : will store the content to the object’s path, either on filesystem or on Swift object storage + +=item B : the db accessor object + +=back + =cut +sub store { + my $c = shift; + my $upload = shift; + + if ($c->app->config('swift')) { + $c->app->swift->put_object( + container_name => $c->app->config('swift')->{container}, + object_name => $c->path, + content_length => $upload->size, + content => $upload->slurp + ); + } else { + $upload->move_to($c->path); + } + + return $c; +} + +=head2 retrieve + +=over 1 + +=item B : C<$c-Eretrieve> + +=item B : none + +=item B : get file from storage, either filesystem or Swift object storage + +=item B : the data from the file + +=back + +=cut + +sub retrieve { + my $c = shift; + my $upload = shift; + + if ($c->app->config('swift')) { + my $file; + $c->app->swift->get_object( + container_name => $c->app->config('swift')->{container}, + object_name => $c->path, + write_code => sub { + my ($status, $message, $headers, $chunk) = @_; + $file .= $chunk; + } + ); + return $file; + } else { + return Mojo::File->new($c->path)->slurp; + } +} + +=head2 decrypt + +=over 1 + +=item B : C<$c-Edecrypt($key)> + +=item B : the decryption key + +=item B : decrypt the image + +=item B : a Mojo::Asset::File object + +=back + +=cut + +sub decrypt { + my $c = shift; + my $key = shift; + $c->iv = 'dupajasi' unless $c->iv; + + my $cipher = Crypt::CBC->new( + -key => $key, + -cipher => 'Blowfish', + -header => 'none', + -iv => $c->iv + ); + + $cipher->start('decrypting'); + + my $decrypt_asset = Mojo::Asset::File->new; + + if ($c->app->config('swift')) { + $c->app->swift->get_object( + container_name => $c->app->config('swift')->{container}, + object_name => $c->path, + write_code => sub { + my ($status, $message, $headers, $chunk) = @_; + $decrypt_asset->add_chunk($cipher->crypt($chunk)); + } + ); + } else { + open(my $f, "<", $c->path) or die "Unable to read encrypted file: $!"; + binmode $f; + while (read($f, my $buffer, 1024)) { + $decrypt_asset->add_chunk($cipher->crypt($buffer)); + } + $decrypt_asset->add_chunk($cipher->finish); + + } + return $decrypt_asset->slurp; +} + +=head2 delete + +=over 1 + +=item B : C<$c-Edelete> + +=item B : none + +=item B : delete the file on filesystem or Swift object storage and disable the image in database + +=item B : the db accessor object + +=back + +=cut + +sub delete { + my $c = shift; + if ($c->app->config('cache_max_size') != 0 || scalar(@{$c->app->config('memcached_servers')})) { + $c->app->chi('lutim_images_cache')->remove($c->short); + } + if ($c->app->config('swift')) { + $c->app->swift->delete_object({ + container_name => $c->app->config('swift')->{container}, + object_name => $c->path + }); + } else { + unlink $c->path or warn "Could not unlink ".$c->path.": $!"; + } + return $c->disable(); +} 1; diff --git a/lib/Lutim/Plugin/Helpers.pm b/lib/Lutim/Plugin/Helpers.pm index 3ea009e..1bec2be 100644 --- a/lib/Lutim/Plugin/Helpers.pm +++ b/lib/Lutim/Plugin/Helpers.pm @@ -49,26 +49,61 @@ sub register { $app->helper(is_selected => \&_is_selected); $app->helper(is_wm_selected => \&_is_wm_selected); $app->helper(crypt => \&_crypt); - $app->helper(decrypt => \&_decrypt); - $app->helper(delete_image => \&_delete_image); $app->helper(iso639_native_name => \&_iso639_native_name); $app->helper(prefix => \&_prefix); + if ($app->config('swift')) { + use Net::OpenStack::Swift; + $app->helper(swift => \&_swift); + $app->helper(check_swift_container => \&_check_swift_container); + $app->helper(find_swift_container => \&_find_swift_container); + } } sub _pg { - my $c = shift; + my $c = shift; state $pg = Mojo::Pg->new($c->app->pg_url($c->app->config('pgdb'))); return $pg; } sub _sqlite { - my $c = shift; + my $c = shift; state $sqlite = Mojo::SQLite->new('sqlite:'.$c->app->config('db_path')); return $sqlite; } +sub _swift { + my $c = shift; + + state $swift = Net::OpenStack::Swift->new($c->app->config('swift')); + return $swift; +} + +sub _check_swift_container { + my $c = shift; + + my ($storage_url, $token) = $c->swift->get_auth(); + my ($headers, $containers) = $c->swift->get_account(url => $storage_url, token => $token); + unless ($c->find_swift_container($containers)) { + $c->swift->put_container(container_name => $c->app->config('swift')->{container}); + my ($headers, $containers) = $c->swift->get_account(url => $storage_url, token => $token); + die sprintf("Swift container %s not found, and unable to create it.", $c->app->config('swift')->{container}) unless $c->find_swift_container($containers); + } +} + +sub _find_swift_container { + my $c = shift; + my $containers = shift; + + my $found_container = 0; + foreach my $container (@{$containers}) { + $found_container = 1 if $container->{name} eq $c->app->config('swift')->{container}; + } + + return $found_container; +} + sub _render_file { my $c = shift; my ($im_loaded, $img, $dl, $key, $thumb) = @_; @@ -83,12 +118,15 @@ sub _render_file { $dl = 'attachment' if ($mediatype =~ m/svg/); $filename = quote($filename); - unless (-f $path && -r $path) { - $c->app->log->error("Cannot read file [$path]. error [$!]"); - $c->flash( - msg => $c->l('Unable to find the image: it has been deleted.') - ); - return 500; + unless ($c->app->config('swift')) { + # Do we have the file on disk? + unless (-f $path && -r $path) { + $c->app->log->error("Cannot read file [$path]. error [$!]"); + $c->flash( + msg => $c->l('Unable to find the image: it has been deleted.') + ); + return 500; + } } $mediatype =~ s/x-//; @@ -104,21 +142,22 @@ sub _render_file { $c->res->content->headers($headers); my $cache; + # Do we have the file in cache? if ($c->config('cache_max_size') != 0 || scalar(@{$c->config('memcached_servers')})) { $cache = $c->chi('lutim_images_cache')->compute($img->short, undef, sub { if ($key) { return { - asset => $c->decrypt($key, $path, $iv), + asset => $img->decrypt($key), key => $key }; } else { return { - asset => Mojo::File->new($path)->slurp, + asset => $img->retrieve }; } }); if ($key && $key ne $cache->{key}) { - my $tmp = $c->decrypt($key, $path, $iv); + my $tmp = $img->decrypt($key); $cache->{asset} = $tmp; $c->chi('lutim_images_cache')->replace( $img->short, @@ -131,18 +170,18 @@ sub _render_file { } else { if ($key) { $cache = { - asset => $c->decrypt($key, $path, $iv), + asset => $img->decrypt($key), }; } else { $cache = { - asset => Mojo::File->new($path)->slurp, + asset => $img->retrieve, }; } } - # Extend expiration time my $asset = Mojo::Asset::Memory->new; $asset->add_chunk($cache->{asset}); + # Do we need to create a thumbnail? if (defined $thumb && $im_loaded && $mediatype ne 'image/svg+xml' && $mediatype !~ m#image/(x-)?xcf# && $mediatype ne 'image/webp') { # ImageMagick don't work in Debian with svg (for now?) my $im = Image::Magick->new; $im->BlobToImage($asset->slurp); @@ -287,44 +326,6 @@ sub _crypt { return ($crypt_upload, $key, $iv); } -sub _decrypt { - my $c = shift; - my $key = shift; - my $file = shift; - my $iv = shift; - $iv = 'dupajasi' unless $iv; - - my $cipher = Crypt::CBC->new( - -key => $key, - -cipher => 'Blowfish', - -header => 'none', - -iv => $iv - ); - - $cipher->start('decrypting'); - - my $decrypt_asset = Mojo::Asset::File->new; - - open(my $f, "<",$file) or die "Unable to read encrypted file: $!"; - binmode $f; - while (read($f, my $buffer, 1024)) { - $decrypt_asset->add_chunk($cipher->crypt($buffer)); - } - $decrypt_asset->add_chunk($cipher->finish) ; - - return $decrypt_asset->slurp; -} - -sub _delete_image { - my $c = shift; - my $img = shift; - if ($c->config('cache_max_size') != 0 || scalar(@{$c->config('memcached_servers')})) { - $c->chi('lutim_images_cache')->remove($img->short); - } - unlink $img->path or warn "Could not unlink ".$img->path.": $!"; - $img->disable(); -} - sub _iso639_native_name { my $c = shift; return ucfirst(decode 'UTF-8', get_iso639_1(shift)->{nativeName}); diff --git a/lutim.conf.template b/lutim.conf.template index 8c3d12f..3521062 100644 --- a/lutim.conf.template +++ b/lutim.conf.template @@ -205,6 +205,18 @@ # optional, default is 3600 #session_duration => 3600, + # you can store images on Swift object storage (https://en.wikipedia.org/wiki/OpenStack#Swift) instead of filesystem + # please read https://metacpan.org/pod/Net::OpenStack::Swift#SYNOPSIS to know how to configure this setting + # IMPORTANT: add a `container` key in it, to let Lutim know which container to use. This is not a regular Net::OpenStack::Swift setting, but Lutim need it. + # optional, no default + #swift => { + # auth_url => 'https://auth-endpoint-url/v2.0', + # user => 'userid', + # password => 'password', + # tenant_name => 'project_id', + # container => 'lutim' + #}, + # disable counters of images # set to 1 to disable counters # optional, counters are enabled by default diff --git a/themes/default/lib/Lutim/I18N/en.po b/themes/default/lib/Lutim/I18N/en.po index 4ae2101..fb16312 100644 --- a/themes/default/lib/Lutim/I18N/en.po +++ b/themes/default/lib/Lutim/I18N/en.po @@ -31,11 +31,11 @@ msgstr "%1 sent images on this instance from beginning." msgid "-or-" msgstr "-or-" -#: lib/Lutim.pm:342 lib/Lutim/Command/cron/stats.pm:158 lib/Lutim/Command/cron/stats.pm:172 lib/Lutim/Command/cron/stats.pm:189 themes/default/templates/index.html.ep:5 themes/default/templates/myfiles.html.ep:5 themes/default/templates/partial/raw.js.ep:25 themes/default/templates/partial/raw.js.ep:8 themes/default/templates/raw.html.ep:10 +#: lib/Lutim.pm:347 lib/Lutim/Command/cron/stats.pm:158 lib/Lutim/Command/cron/stats.pm:172 lib/Lutim/Command/cron/stats.pm:189 themes/default/templates/index.html.ep:5 themes/default/templates/myfiles.html.ep:5 themes/default/templates/partial/raw.js.ep:25 themes/default/templates/partial/raw.js.ep:8 themes/default/templates/raw.html.ep:10 msgid "1 year" msgstr "1 year" -#: lib/Lutim.pm:341 lib/Lutim/Command/cron/stats.pm:155 lib/Lutim/Command/cron/stats.pm:169 lib/Lutim/Command/cron/stats.pm:186 themes/default/templates/index.html.ep:4 themes/default/templates/myfiles.html.ep:33 themes/default/templates/myfiles.html.ep:4 themes/default/templates/partial/for_my_delay.html.ep:13 themes/default/templates/partial/lutim.js.ep:149 themes/default/templates/partial/raw.js.ep:22 themes/default/templates/partial/raw.js.ep:5 themes/default/templates/raw.html.ep:7 +#: lib/Lutim.pm:346 lib/Lutim/Command/cron/stats.pm:155 lib/Lutim/Command/cron/stats.pm:169 lib/Lutim/Command/cron/stats.pm:186 themes/default/templates/index.html.ep:4 themes/default/templates/myfiles.html.ep:33 themes/default/templates/myfiles.html.ep:4 themes/default/templates/partial/for_my_delay.html.ep:13 themes/default/templates/partial/lutim.js.ep:149 themes/default/templates/partial/raw.js.ep:22 themes/default/templates/partial/raw.js.ep:5 themes/default/templates/raw.html.ep:7 msgid "24 hours" msgstr "24 hours" @@ -211,7 +211,7 @@ msgstr "Image delay" msgid "Image deleted" msgstr "Image deleted" -#: lib/Lutim/Controller/Image.pm:799 +#: lib/Lutim/Controller/Image.pm:804 msgid "Image not found." msgstr "Image not found." @@ -369,7 +369,7 @@ msgid "Something bad happened" msgstr "Something bad happened" #. ($c->config('contact') -#: lib/Lutim/Controller/Image.pm:806 +#: lib/Lutim/Controller/Image.pm:811 msgid "Something went wrong when creating the zip file. Try again later or contact the administrator (%1)." msgstr "Something went wrong when creating the zip file. Try again later or contact the administrator (%1)." @@ -398,7 +398,7 @@ msgid "The delete token is invalid." msgstr "The delete token is invalid." #. ($upload->filename) -#: lib/Lutim/Controller/Image.pm:531 +#: lib/Lutim/Controller/Image.pm:532 msgid "The file %1 is not an image." msgstr "The file %1 is not an image." @@ -440,7 +440,7 @@ msgid "There is XXXX image(s) in the gallery" msgstr "There is XXXX image(s) in the gallery" #. ($c->config->{contact}) -#: lib/Lutim/Controller/Image.pm:528 +#: lib/Lutim/Controller/Image.pm:529 msgid "There is no more available URL. Retry or contact the administrator. %1" msgstr "There is no more available URL. Retry or contact the administrator. %1" @@ -473,7 +473,7 @@ msgstr "Unable to copy to clipboard" msgid "Unable to find the image %1." msgstr "Unable to find the image %1." -#: lib/Lutim/Controller/Image.pm:617 lib/Lutim/Controller/Image.pm:662 lib/Lutim/Controller/Image.pm:703 lib/Lutim/Controller/Image.pm:746 lib/Lutim/Controller/Image.pm:758 lib/Lutim/Controller/Image.pm:769 lib/Lutim/Controller/Image.pm:796 lib/Lutim/Plugin/Helpers.pm:89 +#: lib/Lutim/Controller/Image.pm:619 lib/Lutim/Controller/Image.pm:665 lib/Lutim/Controller/Image.pm:706 lib/Lutim/Controller/Image.pm:749 lib/Lutim/Controller/Image.pm:761 lib/Lutim/Controller/Image.pm:773 lib/Lutim/Controller/Image.pm:801 lib/Lutim/Plugin/Helpers.pm:126 msgid "Unable to find the image: it has been deleted." msgstr "Unable to find the image: it has been deleted." @@ -498,7 +498,7 @@ msgid "Uploaded files by days" msgstr "Uploaded files by days" #. ($c->app->config('contact') -#: lib/Lutim/Plugin/Helpers.pm:222 +#: lib/Lutim/Plugin/Helpers.pm:261 msgid "Uploading is currently disabled, please try later or contact the administrator (%1)." msgstr "Uploading is currently disabled, please try later or contact the administrator (%1)." @@ -558,7 +558,7 @@ msgstr "arabic translation" msgid "core developer" msgstr "core developer" -#: lib/Lutim.pm:340 lib/Lutim/Command/cron/stats.pm:154 lib/Lutim/Command/cron/stats.pm:168 lib/Lutim/Command/cron/stats.pm:185 themes/default/templates/index.html.ep:3 themes/default/templates/myfiles.html.ep:3 themes/default/templates/partial/raw.js.ep:21 themes/default/templates/partial/raw.js.ep:4 themes/default/templates/raw.html.ep:6 +#: lib/Lutim.pm:345 lib/Lutim/Command/cron/stats.pm:154 lib/Lutim/Command/cron/stats.pm:168 lib/Lutim/Command/cron/stats.pm:185 themes/default/templates/index.html.ep:3 themes/default/templates/myfiles.html.ep:3 themes/default/templates/partial/raw.js.ep:21 themes/default/templates/partial/raw.js.ep:4 themes/default/templates/raw.html.ep:6 msgid "no time limit" msgstr "no time limit" diff --git a/themes/default/lib/Lutim/I18N/lutim.pot b/themes/default/lib/Lutim/I18N/lutim.pot index 96a2f26..22d901a 100644 --- a/themes/default/lib/Lutim/I18N/lutim.pot +++ b/themes/default/lib/Lutim/I18N/lutim.pot @@ -32,11 +32,11 @@ msgstr "" msgid "-or-" msgstr "" -#: lib/Lutim.pm:342 lib/Lutim/Command/cron/stats.pm:158 lib/Lutim/Command/cron/stats.pm:172 lib/Lutim/Command/cron/stats.pm:189 themes/default/templates/index.html.ep:5 themes/default/templates/myfiles.html.ep:5 themes/default/templates/partial/raw.js.ep:25 themes/default/templates/partial/raw.js.ep:8 themes/default/templates/raw.html.ep:10 +#: lib/Lutim.pm:347 lib/Lutim/Command/cron/stats.pm:158 lib/Lutim/Command/cron/stats.pm:172 lib/Lutim/Command/cron/stats.pm:189 themes/default/templates/index.html.ep:5 themes/default/templates/myfiles.html.ep:5 themes/default/templates/partial/raw.js.ep:25 themes/default/templates/partial/raw.js.ep:8 themes/default/templates/raw.html.ep:10 msgid "1 year" msgstr "" -#: lib/Lutim.pm:341 lib/Lutim/Command/cron/stats.pm:155 lib/Lutim/Command/cron/stats.pm:169 lib/Lutim/Command/cron/stats.pm:186 themes/default/templates/index.html.ep:4 themes/default/templates/myfiles.html.ep:33 themes/default/templates/myfiles.html.ep:4 themes/default/templates/partial/for_my_delay.html.ep:13 themes/default/templates/partial/lutim.js.ep:149 themes/default/templates/partial/raw.js.ep:22 themes/default/templates/partial/raw.js.ep:5 themes/default/templates/raw.html.ep:7 +#: lib/Lutim.pm:346 lib/Lutim/Command/cron/stats.pm:155 lib/Lutim/Command/cron/stats.pm:169 lib/Lutim/Command/cron/stats.pm:186 themes/default/templates/index.html.ep:4 themes/default/templates/myfiles.html.ep:33 themes/default/templates/myfiles.html.ep:4 themes/default/templates/partial/for_my_delay.html.ep:13 themes/default/templates/partial/lutim.js.ep:149 themes/default/templates/partial/raw.js.ep:22 themes/default/templates/partial/raw.js.ep:5 themes/default/templates/raw.html.ep:7 msgid "24 hours" msgstr "" @@ -212,7 +212,7 @@ msgstr "" msgid "Image deleted" msgstr "" -#: lib/Lutim/Controller/Image.pm:799 +#: lib/Lutim/Controller/Image.pm:804 msgid "Image not found." msgstr "" @@ -370,7 +370,7 @@ msgid "Something bad happened" msgstr "" #. ($c->config('contact') -#: lib/Lutim/Controller/Image.pm:806 +#: lib/Lutim/Controller/Image.pm:811 msgid "Something went wrong when creating the zip file. Try again later or contact the administrator (%1)." msgstr "" @@ -399,7 +399,7 @@ msgid "The delete token is invalid." msgstr "" #. ($upload->filename) -#: lib/Lutim/Controller/Image.pm:531 +#: lib/Lutim/Controller/Image.pm:532 msgid "The file %1 is not an image." msgstr "" @@ -441,7 +441,7 @@ msgid "There is XXXX image(s) in the gallery" msgstr "" #. ($c->config->{contact}) -#: lib/Lutim/Controller/Image.pm:528 +#: lib/Lutim/Controller/Image.pm:529 msgid "There is no more available URL. Retry or contact the administrator. %1" msgstr "" @@ -474,7 +474,7 @@ msgstr "" msgid "Unable to find the image %1." msgstr "" -#: lib/Lutim/Controller/Image.pm:617 lib/Lutim/Controller/Image.pm:662 lib/Lutim/Controller/Image.pm:703 lib/Lutim/Controller/Image.pm:746 lib/Lutim/Controller/Image.pm:758 lib/Lutim/Controller/Image.pm:769 lib/Lutim/Controller/Image.pm:796 lib/Lutim/Plugin/Helpers.pm:89 +#: lib/Lutim/Controller/Image.pm:619 lib/Lutim/Controller/Image.pm:665 lib/Lutim/Controller/Image.pm:706 lib/Lutim/Controller/Image.pm:749 lib/Lutim/Controller/Image.pm:761 lib/Lutim/Controller/Image.pm:773 lib/Lutim/Controller/Image.pm:801 lib/Lutim/Plugin/Helpers.pm:126 msgid "Unable to find the image: it has been deleted." msgstr "" @@ -499,7 +499,7 @@ msgid "Uploaded files by days" msgstr "" #. ($c->app->config('contact') -#: lib/Lutim/Plugin/Helpers.pm:222 +#: lib/Lutim/Plugin/Helpers.pm:261 msgid "Uploading is currently disabled, please try later or contact the administrator (%1)." msgstr "" @@ -559,7 +559,7 @@ msgstr "" msgid "core developer" msgstr "" -#: lib/Lutim.pm:340 lib/Lutim/Command/cron/stats.pm:154 lib/Lutim/Command/cron/stats.pm:168 lib/Lutim/Command/cron/stats.pm:185 themes/default/templates/index.html.ep:3 themes/default/templates/myfiles.html.ep:3 themes/default/templates/partial/raw.js.ep:21 themes/default/templates/partial/raw.js.ep:4 themes/default/templates/raw.html.ep:6 +#: lib/Lutim.pm:345 lib/Lutim/Command/cron/stats.pm:154 lib/Lutim/Command/cron/stats.pm:168 lib/Lutim/Command/cron/stats.pm:185 themes/default/templates/index.html.ep:3 themes/default/templates/myfiles.html.ep:3 themes/default/templates/partial/raw.js.ep:21 themes/default/templates/partial/raw.js.ep:4 themes/default/templates/raw.html.ep:6 msgid "no time limit" msgstr ""