feature 2548 multisize - added a page to build missing derivatives

- browser driven, chained ws calls to retrieve urls, visual feedback of progress through slideshow

git-svn-id: http://piwigo.org/svn/trunk@12865 68402e56-0260-453c-a942-63ccdbb3a9ee

admin/derivatives_build.php

@@ -0,0 +1,30 @@
+<?php
+// +-----------------------------------------------------------------------+
+// | Piwigo - a PHP based photo gallery                                    |
+// +-----------------------------------------------------------------------+
+// | Copyright(C) 2008-2011 Piwigo Team                  http://piwigo.org |
+// | Copyright(C) 2003-2008 PhpWebGallery Team    http://phpwebgallery.net |
+// | Copyright(C) 2002-2003 Pierrick LE GALL   http://le-gall.net/pierrick |
+// +-----------------------------------------------------------------------+
+// | This program is free software; you can redistribute it and/or modify  |
+// | it under the terms of the GNU General Public License as published by  |
+// | the Free Software Foundation                                          |
+// |                                                                       |
+// | This program is distributed in the hope that it will be useful, but   |
+// | WITHOUT ANY WARRANTY; without even the implied warranty of            |
+// | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU      |
+// | General Public License for more details.                              |
+// |                                                                       |
+// | You should have received a copy of the GNU General Public License     |
+// | along with this program; if not, write to the Free Software           |
+// | Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, |
+// | USA.                                                                  |
+// +-----------------------------------------------------------------------+
+
+defined('PHPWG_ROOT_PATH') or trigger_error('Hacking attempt!', E_USER_ERROR);
+
+$template->assign('derivatives', array_keys(ImageStdParams::get_defined_type_map()));
+
+$template->set_filename('derivatives', 'derivatives_build.tpl');
+$template->assign_var_from_handle('ADMIN_CONTENT', 'derivatives');
+?>

admin/include/functions.php

@@ -200,11 +200,8 @@ SELECT
     $files = array();
     $files[] = get_element_path($row);
 
-   
     if (!empty($row['representative_ext']))
     {
-      $pi = pathinfo($row['path']);
-      $file_wo_ext = get_filename_wo_extension($pi['basename']);
       $files[] = original_to_representative( $files[0], $row['representative_ext']);
     }
 

admin/include/image.class.php

@@ -704,6 +704,7 @@ class image_gd implements imageInterface
 		// Place the source image in the destination image
 		imagecopy($cut, $ioverlay, 0, 0, 0, 0, $ow, $oh);
 		imagecopymerge($this->image, $cut, $x, $y, 0, 0, $ow, $oh, $opacity);
+    imagedestroy($cut);
     return true;
   }
 

admin/themes/default/template/derivatives.tpl

@@ -22,6 +22,7 @@
 </style>
 {/literal}{/html_head}
 
+<p><a href="admin.php?page=derivatives_build">Build missing derivatives</a></p>
 <form method="post" id="derviativesForm">
 <fieldset>
 <legend>{'Watermark'|@translate}</legend>

admin/themes/default/template/derivatives_build.tpl

@@ -0,0 +1,152 @@
+{html_head}{literal}
+<style type="text/css">
+TABLE {
+	font-size: larger;
+}
+</style>
+{/literal}{/html_head}
+
+<p>
+	<select id="types" name="types[]" multiple="multiple">
+	{foreach from=$derivatives item=type}
+	<option value="{$type}" selected="selected">{$type|@translate}</option>
+	{/foreach}
+	</select>
+	<input id="startLink" value="{'Start'|@translate}" onclick="start()" type="button">
+	<input id="pauseLink" value="{'Pause'|@translate}" onclick="pause()" type="button" disabled="disbled">
+	<input id="stopLink" value="{'Stop'|@translate}" onclick="stop()" type="button" disabled="disbled">
+</p>
+<hr/>
+<p>
+<table>
+	<tr>
+		<td>Errors</td>
+		<td id="errors">0</td>
+	</tr>
+	<tr>
+		<td>Loaded</td>
+		<td id="loaded">0</td>
+	</tr>
+	<tr>
+		<td>Remaining</td>
+		<td id="remaining">0</td>
+	</tr>
+</table>
+<div id="feedbackWrap" style="height:320px; min-height:320px;">
+<img id="feedbackImg">
+</div>
+</p>
+
+<div id="errorList">
+</div>
+
+{combine_script id='iloader' load='footer' path='themes/default/js/image.loader.js'}
+
+{footer_script require='jquery.effects.slide'}{literal}
+
+var loader = new ImageLoader( {onChanged: loaderChanged} )
+	, pending_next_page = null
+	, last_image_show_time = 0
+	, allDoneDfd, urlDfd;
+
+function start() {
+	allDoneDfd = jQuery.Deferred();
+	urlDfd = jQuery.Deferred();
+
+	allDoneDfd.always( function() {
+			jQuery("#startLink").attr('disabled', false).css("opacity", 1);
+			jQuery("#pauseLink,#stopLink").attr('disabled', true).css("opacity", 0.5);
+		} );
+
+	urlDfd.always( function() {
+		if (loader.remaining()==0)
+			allDoneDfd.resolve();
+		} );
+
+	jQuery("#startLink").attr('disabled', true).css("opacity", 0.5);
+	jQuery("#pauseLink,#stopLink").attr('disabled', false).css("opacity", 1);
+
+	loader.pause(false);
+	updateStats();
+	getUrls();
+}
+
+function pause() {
+	loader.pause( !loader.pause() );
+}
+
+function stop() {
+	loader.clear();
+	urlDfd.resolve();
+}
+
+function getUrls(page_token) {
+	data = {max_urls: 500, types: []};
+	jQuery.each(jQuery("#types").serializeArray(), function(i, t) {
+			data.types.push( t.value );
+		} );
+
+	if (page_token)
+		data['prev_page'] = page_token;
+	jQuery.post( '{/literal}{$ROOT_URL}{literal}ws.php?format=json&method=pwg.getMissingDerivatives',
+		data, wsData, "json").fail( wsError );
+}
+
+function wsData(data) {
+	if (!data.stat || data.stat != "ok") {
+		wsError();
+		return;
+	}
+	loader.add( data.result.urls );
+	if (data.result.next_page) {
+		if (loader.pause() || loader.remaining() > 100) {
+			pending_next_page = data.result.next_page;
+		}
+		else {
+			getUrls(data.result.next_page);
+		}
+	}
+}
+
+function wsError() {
+	urlDfd.reject();
+}
+
+function updateStats() {
+	jQuery("#loaded").text( loader.loaded );
+	jQuery("#errors").text( loader.errors );
+	jQuery("#remaining").text( loader.remaining() );
+}
+
+function loaderChanged(type, img) {
+	updateStats();
+	if (img) {
+		if (type==="load") {
+			var now = jQuery.now();
+			if (now - last_image_show_time > 3000) {
+				last_image_show_time = now;
+				var h=img.height, url=img.src;
+				jQuery("#feedbackWrap").hide("slide", {direction:'down'}, function() {
+					last_image_show_time = jQuery.now();
+					if (h > 300 )
+						jQuery("#feedbackImg").attr("height", 300);
+					else
+						jQuery("#feedbackImg").removeAttr("height");
+					jQuery("#feedbackImg").attr("src", url);
+					jQuery("#feedbackWrap").show("slide", {direction:'up'} );
+					} );
+			}
+		}
+		else {
+			jQuery("#errorList").prepend( '<a href="'+img.src+'">'+img.src+'</a>' + "<br>");
+		}
+	}
+	if (pending_next_page && 100 > loader.remaining() )	{
+		getUrls(pending_next_page);
+		pending_next_page = null;
+	}
+	else if (loader.remaining() == 0 && (urlDfd.isResolved() || urlDfd.isRejected()))	{
+		allDoneDfd.resolve();
+	}
+}
+{/literal}{/footer_script}

i.php

@@ -108,6 +108,10 @@ function parse_request()
   else
   {
     $req = $_SERVER["QUERY_STRING"];
+    if ($pos=strpos($req, '&'))
+    {
+      $req = substr($req, 0, $pos);
+    }
     /*foreach (array_keys($_GET) as $keynum => $key)
     {
       $req = $key;
@@ -167,12 +171,20 @@ function parse_request()
   {
     try
     {
-      $page['derivative_params'] = DerivativeParams::from_url_tokens($deriv);
+      $params = $page['derivative_params'] = DerivativeParams::from_url_tokens($deriv);
     }
     catch (Exception $e)
     {
       ierror($e->getMessage(), 400);
     }
+    if ($params->sizing->ideal_size[0] < 20 or $params->sizing->ideal_size[1] < 20)
+    {
+      ierror('Invalid size', 400);
+    }
+    if ($params->sizing->max_crop < 0 or $params->sizing->max_crop > 1)
+    {
+      ierror('Invalid crop', 400);
+    }
   }
 
   if ($req[0]!='g' && $req[0]!='u')
@@ -224,14 +236,6 @@ parse_request();
 //var_export($page);
 
 $params = $page['derivative_params'];
-if ($params->sizing->ideal_size[0] < 20 or $params->sizing->ideal_size[1] < 20)
-{
-  ierror('Invalid size', 400);
-}
-if ($params->sizing->max_crop < 0 or $params->sizing->max_crop > 1)
-{
-  ierror('Invalid crop', 400);
-}
 
 $src_mtime = @filemtime($page['src_path']);
 if ($src_mtime === false)
@@ -250,7 +254,12 @@ if ($derivative_mtime === false or
 
 $expires=false;
 $now = time();
-if ( $now > (max($src_mtime, $params->last_mod_time) + 24*3600) )
+if ( isset($_GET['b']) )
+{
+  $expires = $now + 100;
+  header("Cache-control: no-store, max-age=100");
+}
+elseif ( $now > (max($src_mtime, $params->last_mod_time) + 24*3600) )
 {// somehow arbitrary - if derivative params or src didn't change for the last 24 hours, we send an expire header for several days
   $expires = $now + 10*24*3600;
 }
@@ -267,6 +276,10 @@ if (!$need_generate)
   send_derivative($expires);
 }
 
+if (!mkgetdir(dirname($page['derivative_path'])))
+{
+  ierror("dir create error", 500);
+}
 
 include_once(PHPWG_ROOT_PATH . 'admin/include/image.class.php');
 
@@ -275,11 +288,6 @@ set_time_limit(0);
 
 $image = new pwg_image($page['src_path']);
 
-if (!mkgetdir(dirname($page['derivative_path'])))
-{
-  ierror("dir create error", 500);
-}
-
 $changes = 0;
 
 // todo rotate
@@ -325,6 +333,15 @@ if ($params->use_watermark)
     if ($wm->xrepeat)
     {
       // todo
+      $pad = $wm_size[0] + max(30, round($wm_size[0]/4));
+      for($i=-$wm->xrepeat; $i<=$wm->xrepeat; $i++)
+      {
+        if (!$i) continue;
+        $x2 = $x + $i * $pad;
+        if ($x2>=0 && $x2+$wm_size[0]<$d_size[0])
+          if (!$image->compose($wm_image, $x2, $y, $wm->opacity))
+            break;
+      }
     }
   }
   $wm_image->destroy();

include/derivative.inc.php

@@ -219,6 +219,11 @@ final class DerivativeImage
     }
   }
 
+  function get_path()
+  {
+    return PHPWG_ROOT_PATH.$this->rel_path;
+  }
+
   function get_url()
   {
     return get_root_url().$this->rel_url;

include/ws_functions.inc.php

@@ -92,9 +92,9 @@ function ws_std_image_sql_filter( $params, $tbl_name='' )
   {
     $clauses[] = $tbl_name.'width/'.$tbl_name.'height<='.$params['f_max_ratio'];
   }
-  if ( $params['f_with_thumbnail'] )
+  if (is_numeric($params['f_max_level']) )
   {
-    $clauses[] = $tbl_name.'tn_ext IS NOT NULL';
+    $clauses[] = $tbl_name.'level <= '.$params['f_max_level'];
   }
   return $clauses;
 }
@@ -179,6 +179,92 @@ function ws_std_get_image_xml_attributes()
     );
 }
 
+function ws_getMissingDerivatives($params, &$service)
+{
+  if (!is_admin())
+  {
+    return new PwgError(403, 'Forbidden');
+  }
+
+  if ( empty($params['types']) )
+  {
+    $types = array_keys(ImageStdParams::get_defined_type_map());
+  }
+  else
+  {
+    $types = array_intersect(array_keys(ImageStdParams::get_defined_type_map()), $params['types']);
+    if (count($types)==0)
+    {
+      return new PwgError(WS_ERR_INVALID_PARAM, "Invalid types");
+    }
+  }
+
+  if ( ($max_urls = intval($params['max_urls'])) <= 0)
+  {
+    return new PwgError(WS_ERR_INVALID_PARAM, "Invalid max_urls");
+  }
+
+  list($max_id, $image_count) = pwg_db_fetch_row( pwg_query('SELECT MAX(id)+1, COUNT(*) FROM '.IMAGES_TABLE) );
+  $start_id = intval($params['prev_page']);
+  if ($start_id<=0)
+  {
+    $start_id = $max_id;
+  }
+
+  $uid = '&b='.time();
+  global $conf;
+  $conf['question_mark_in_urls'] = $conf['php_extension_in_urls'] = true;
+  $conf['derivative_url_style']=2; //script
+
+  $qlimit = min(5000, ceil(max($image_count/500, $max_urls/count($types))));
+  $where_clauses = ws_std_image_sql_filter( $params, '' );
+  $where_clauses[] = 'id<start_id';
+
+  $query_model = 'SELECT id, path, representative_ext, width, height
+  FROM '.IMAGES_TABLE.'
+  WHERE '.implode(' AND ', $where_clauses).'
+  ORDER BY id DESC
+  LIMIT '.$qlimit;
+
+  $urls=array();
+  do
+  {
+    $result = pwg_query( str_replace('start_id', $start_id, $query_model));
+    $is_last = pwg_db_num_rows($result) < $qlimit;
+    while ($row=pwg_db_fetch_assoc($result))
+    {
+      $start_id = $row['id'];
+      $src_image = new SrcImage($row);
+      if ($src_image->is_mimetype())
+        continue;
+      foreach($types as $type)
+      {
+        $derivative = new DerivativeImage($type, $src_image);
+        if ($type != $derivative->get_type())
+          continue;
+        if (@filemtime($derivative->get_path())===false)
+        {
+          $urls[] = $derivative->get_url().$uid;
+        }
+      }
+      if (count($urls)>=$max_urls && !$is_last)
+        break;
+    }
+    if ($is_last)
+    {
+      $start_id = 0;
+    }
+  }while (count($urls)<$max_urls && $start_id);
+
+  $ret = array();
+  if ($start_id)
+  {
+    $ret['next_page']=$start_id;
+  }
+  $ret['urls']=$urls;
+  return $ret;
+}
+
 /**
  * returns PWG version (web service method)
  */

themes/default/js/image.loader.js

@@ -0,0 +1,84 @@
+
+function ImageLoader(opts) {
+	this.opts = jQuery.extend( {
+			maxRequests: 6,
+			onChanged: jQuery.noop
+		}, opts||{} );
+}
+
+ImageLoader.prototype = {
+	loaded: 0,
+	errors: 0,
+	errorEma: 0,
+
+	pause: false,
+
+	current: [],
+	queue: [],
+	pool: [],
+
+	remaining: function() {
+		return this.current.length + this.queue.length;
+	},
+
+	add: function(urls) {
+		this.queue = this.queue.concat( urls );
+		this._fireChanged("add");
+		this._checkQueue();
+	},
+
+	clear: function() {
+		this.queue.length = 0;
+		while (this.current.length)
+			jQuery( this.current.pop() ).unbind();
+		this.loaded = this.errors = this.errorEma = 0;
+	},
+
+	pause: function(val) {
+		if (val !== undefined)
+		{
+			this.paused = val;
+			this._checkQueue();
+		}
+		return this.paused;
+	},
+
+	_checkQueue: function() {
+		while (!this.paused
+			&& this.queue.length
+			&& this.current.length < this.opts.maxRequests)
+		{
+			this._processOne( this.queue.shift() );
+		}
+	},
+
+	_processOne: function(url) {
+		var img = this.pool.shift() || new Image;
+		this.current.push(img);
+		var that = this;
+		jQuery(img).bind( "load error abort", function(e) {
+		//img.onload = function(e) {
+			jQuery(img).unbind();
+			img.onload=null;
+			that.current.splice(jQuery.inArray(img, that.current), 1);
+			if (e.type==="load") {
+				that.loaded++;
+				that.errorEma *= 0.9;
+			}
+			else {
+				that.errors++;
+				that.errorEma++;
+				if (that.errorEma>=20 && that.errorEma<21)
+					that.paused = true;
+			}
+			that._fireChanged(e.type, img);
+			that._checkQueue();
+			that.pool.push(img);
+		} );
+		img.src = url;
+	},
+
+	_fireChanged: function(type, img) {
+		this.opts.onChanged(type, img);
+	}
+}

themes/default/template/picture_content.tpl

@@ -32,13 +32,13 @@ function toggleDerivativeSwitchBox()
 		elt.style.display="none";
 }
 {/literal}{/footer_script}
-<a id="derivativeSwitchLink" onclick="toggleDerivativeSwitchBox()" style="cursor:pointer">{$current.selected_derivative->get_type()|@translate}</a>
+<a id="derivativeSwitchLink" href="javascript:toggleDerivativeSwitchBox()">{$current.selected_derivative->get_type()|@translate}</a>
 <div id="derivativeSwitchBox" onclick="toggleDerivativeSwitchBox()" style="display:none">
 {foreach from=$current.unique_derivatives item=derivative key=derivative_type}
 <a href="javascript:changeImgSrc('{$derivative->get_url()|@escape:javascript}', '{$derivative_type}', '{$derivative->get_type()|@translate|@escape:javascript}')" style="cursor:pointer">{$derivative->get_type()|@translate} ({$derivative->get_size_hr()})</a><br>
 {/foreach}
 {if isset($U_ORIGINAL)}
-<a  href="javascript:phpWGOpenWindow('{$U_ORIGINAL}','xxx','scrollbars=yes,toolbar=no,status=no,resizable=yes')" title="{'Click on the photo to see it in high definition'|@translate}">{'original'|@translate}</a>
+<a href="javascript:phpWGOpenWindow('{$U_ORIGINAL}','xxx','scrollbars=yes,toolbar=no,status=no,resizable=yes')" title="{'Click on the photo to see it in high definition'|@translate}">{'original'|@translate}</a>
 {/if}
 </div>
 {/if}

ws.php

@@ -69,7 +69,7 @@ function ws_addDefaultMethods( $arr )
         'f_max_date_created' => array( 'default'=> null ),
         'f_min_ratio' => array( 'default'=> null ),
         'f_max_ratio' => array( 'default'=> null ),
-        'f_with_thumbnail' => array( 'default'=> false ),
+        'f_max_level' => array( 'default'=> null ),
       ),
       'Returns elements for the corresponding categories.
 <br><b>cat_id</b> can be empty if <b>recursive</b> is true. Can be sent as an array.
@@ -86,6 +86,25 @@ function ws_addDefaultMethods( $arr )
       ),
       'retrieves a list of categories (tree_output option only compatible with json/php output format' );
 
+  $service->addMethod('pwg.getMissingDerivatives', 'ws_getMissingDerivatives',
+      array(
+        'types' => array( 'default'=>array(), 'flags'=>WS_PARAM_FORCE_ARRAY),
+        'max_urls' => array( 'default' => 200 ),
+        'prev_page' => array( 'default'=> null),
+        'f_min_rate' => array( 'default'=> null ),
+        'f_max_rate' => array( 'default'=> null ),
+        'f_min_hit' => array( 'default'=> null ),
+        'f_max_hit' => array( 'default'=> null ),
+        'f_min_date_available' => array( 'default'=> null ),
+        'f_max_date_available' => array( 'default'=> null ),
+        'f_min_date_created' => array( 'default'=> null ),
+        'f_max_date_created' => array( 'default'=> null ),
+        'f_min_ratio' => array( 'default'=> null ),
+        'f_max_ratio' => array( 'default'=> null ),
+        'f_max_level' => array( 'default'=> null ),
+      ),
+      'retrieves a list of derivatives to build' );
+
   $service->addMethod('pwg.images.addComment', 'ws_images_addComment',
       array(
         'image_id' => array(),
@@ -129,7 +148,7 @@ function ws_addDefaultMethods( $arr )
         'f_max_date_created' => array( 'default'=> null ),
         'f_min_ratio' => array( 'default'=> null ),
         'f_max_ratio' => array( 'default'=> null ),
-        'f_with_thumbnail' => array( 'default'=> false ),
+        'f_max_level' => array( 'default'=> null ),
       ),
       'Returns elements for the corresponding query search.'
     );
@@ -192,7 +211,7 @@ function ws_addDefaultMethods( $arr )
         'f_max_date_created' => array( 'default'=> null ),
         'f_min_ratio' => array( 'default'=> null ),
         'f_max_ratio' => array( 'default'=> null ),
-        'f_with_thumbnail' => array( 'default'=> false ),
+        'f_max_level' => array( 'default'=> null ),
       ),
       'Returns elements for the corresponding tags. Note that tag_id, tag_url_name, tag_name an be arrays. Fill at least one of them. '
     );