Skip to content

Commit 9cc54a0

Browse files
jcupittjcupitt
authored
enable caching of vips_system() (#4640)
* enable caching of vips_system() We were not caching vips_system() calls, since they could potentially have side effects. However, almost all the time we are doing things like running imagemagick commands, so caching is very helpful. This patch enables caching of vips_system(), and adds better hashing of compound argument types. * vips_linear() was changing its args the new arg hasher spotted it improve the arg value change detector * add a "nocache" arg to system * add "cache" arg to system * small polish
1 parent a857538 commit 9cc54a0

7 files changed

Lines changed: 270 additions & 93 deletions

File tree

‎ChangeLog‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ master
77
- larger mmap windows on 64-bit machines improve random access mode for many
88
file formats
99
- pdfload: control region to be rendered via `page_box` [lovell]
10+
- system: add "cache" argument
1011
- add vips_image_get_tile_width(), vips_image_get_tile_height(): get tile
1112
cache geometry hints [jbaiter]
1213

‎libvips/arithmetic/linear.c‎

Lines changed: 27 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -125,62 +125,48 @@ vips_linear_build(VipsObject *object)
125125

126126
int i;
127127

128-
/* If we have a three-element vector, we need to bandup the image to
129-
* match.
130-
*/
131-
linear->n = 1;
132-
if (linear->a)
133-
linear->n = VIPS_MAX(linear->n, linear->a->n);
134-
if (linear->b)
135-
linear->n = VIPS_MAX(linear->n, linear->b->n);
136-
if (unary->in) {
137-
int bands;
138-
139-
vips_image_decode_predict(unary->in, &bands, NULL);
140-
linear->n = VIPS_MAX(linear->n, bands);
141-
}
142-
arithmetic->base_bands = linear->n;
143-
144-
if (unary->in &&
145-
linear->a &&
146-
linear->b) {
147-
if (vips_check_vector(class->nickname,
148-
linear->a->n, unary->in) ||
149-
vips_check_vector(class->nickname,
150-
linear->b->n, unary->in))
151-
return -1;
152-
}
153-
154-
/* If all elements of the constants are equal, we can shrink them down
155-
* to a single element.
128+
/* If all elements of the constants are equal, we can treat them as
129+
* a single element.
156130
*/
131+
int a_n = 1;
157132
if (linear->a) {
158133
double *ary = (double *) linear->a->data;
159-
gboolean all_equal;
160134

161-
all_equal = TRUE;
162135
for (i = 1; i < linear->a->n; i++)
163136
if (ary[i] != ary[0]) {
164-
all_equal = FALSE;
137+
a_n = linear->a->n;
165138
break;
166139
}
167-
168-
if (all_equal)
169-
linear->a->n = 1;
170140
}
141+
int b_n = 1;
171142
if (linear->b) {
172143
double *ary = (double *) linear->b->data;
173-
gboolean all_equal;
174144

175-
all_equal = TRUE;
176145
for (i = 1; i < linear->b->n; i++)
177146
if (ary[i] != ary[0]) {
178-
all_equal = FALSE;
147+
b_n = linear->b->n;
179148
break;
180149
}
150+
}
151+
152+
/* If we have a $n element vector, we need to bandup the image to
153+
* match.
154+
*/
155+
linear->n = 1;
156+
linear->n = VIPS_MAX(linear->n, a_n);
157+
linear->n = VIPS_MAX(linear->n, b_n);
158+
if (unary->in) {
159+
int bands;
181160

182-
if (all_equal)
183-
linear->b->n = 1;
161+
vips_image_decode_predict(unary->in, &bands, NULL);
162+
linear->n = VIPS_MAX(linear->n, bands);
163+
}
164+
arithmetic->base_bands = linear->n;
165+
166+
if (unary->in) {
167+
if (vips_check_vector(class->nickname, a_n, unary->in) ||
168+
vips_check_vector(class->nickname, b_n, unary->in))
169+
return -1;
184170
}
185171

186172
/* Make up-banded versions of our constants.
@@ -191,14 +177,14 @@ vips_linear_build(VipsObject *object)
191177
for (i = 0; i < linear->n; i++) {
192178
if (linear->a) {
193179
double *ary = (double *) linear->a->data;
194-
int j = VIPS_MIN(i, linear->a->n - 1);
180+
int j = VIPS_MIN(i, a_n - 1);
195181

196182
linear->a_ready[i] = ary[j];
197183
}
198184

199185
if (linear->b) {
200186
double *ary = (double *) linear->b->data;
201-
int j = VIPS_MIN(i, linear->b->n - 1);
187+
int j = VIPS_MIN(i, b_n - 1);
202188

203189
linear->b_ready[i] = ary[j];
204190
}

‎libvips/iofuncs/cache.c‎

Lines changed: 162 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,44 @@ vips_value_hash(GParamSpec *pspec, const GValue *value)
185185
else if (generic == G_TYPE_PARAM_BOXED) {
186186
void *p = g_value_get_boxed(value);
187187

188-
return p ? g_direct_hash(p) : 0;
188+
// array_object is an internal type, don't have a case for this
189+
if (!p)
190+
return 0;
191+
else if (G_VALUE_TYPE(value) == VIPS_TYPE_ARRAY_INT) {
192+
int n;
193+
int *array = (int *)
194+
vips_area_get_data(VIPS_AREA(p), NULL, &n, NULL, NULL);
195+
196+
guint hash = 0;
197+
for (int i = 0; i < n; i++)
198+
hash = (hash << 1) ^ g_int_hash(&array[i]);
199+
200+
return hash;
201+
}
202+
else if (G_VALUE_TYPE(value) == VIPS_TYPE_ARRAY_DOUBLE) {
203+
int n;
204+
double *array = (double *)
205+
vips_area_get_data(VIPS_AREA(p), NULL, &n, NULL, NULL);
206+
207+
guint hash = 0;
208+
for (int i = 0; i < n; i++)
209+
hash = (hash << 1) ^ g_double_hash(&array[i]);
210+
211+
return hash;
212+
}
213+
else if (G_VALUE_TYPE(value) == VIPS_TYPE_ARRAY_IMAGE) {
214+
int n;
215+
void **array = (void **)
216+
vips_area_get_data(VIPS_AREA(p), NULL, &n, NULL, NULL);
217+
218+
guint hash = 0;
219+
for (int i = 0; i < n; i++)
220+
hash = (hash << 1) ^ g_direct_hash(array[i]);
221+
222+
return hash;
223+
}
224+
else
225+
return g_direct_hash(p);
189226
}
190227
else if (generic == G_TYPE_PARAM_POINTER) {
191228
void *p = g_value_get_pointer(value);
@@ -275,8 +312,77 @@ vips_value_equal(GParamSpec *pspec, const GValue *v1, const GValue *v2)
275312
else
276313
return s1 && s2 && strcmp(s1, s2) == 0;
277314
}
278-
if (generic == G_TYPE_PARAM_BOXED)
279-
return g_value_get_boxed(v1) == g_value_get_boxed(v2);
315+
if (generic == G_TYPE_PARAM_BOXED) {
316+
void *p1 = g_value_get_boxed(v1);
317+
void *p2 = g_value_get_boxed(v2);
318+
319+
if (p1 == p2)
320+
return TRUE;
321+
else if (!p1 || !p2)
322+
return FALSE;
323+
else if (t1 == VIPS_TYPE_ARRAY_INT) {
324+
int n1;
325+
int *array1 = (int *)
326+
vips_area_get_data(VIPS_AREA(p1), NULL, &n1, NULL, NULL);
327+
int n2;
328+
int *array2 = (int *)
329+
vips_area_get_data(VIPS_AREA(p2), NULL, &n2, NULL, NULL);
330+
331+
if (n1 != n2)
332+
return FALSE;
333+
else if (array1 == array2)
334+
return TRUE;
335+
else {
336+
for (int i = 0; i < n1; i++)
337+
if (array1[i] != array2[i])
338+
return FALSE;
339+
340+
return TRUE;
341+
}
342+
}
343+
else if (t1 == VIPS_TYPE_ARRAY_DOUBLE) {
344+
int n1;
345+
double *array1 = (double *)
346+
vips_area_get_data(VIPS_AREA(p1), NULL, &n1, NULL, NULL);
347+
int n2;
348+
double *array2 = (double *)
349+
vips_area_get_data(VIPS_AREA(p2), NULL, &n2, NULL, NULL);
350+
351+
if (n1 != n2)
352+
return FALSE;
353+
else if (array1 == array2)
354+
return TRUE;
355+
else {
356+
for (int i = 0; i < n1; i++)
357+
if (array1[i] != array2[i])
358+
return FALSE;
359+
360+
return TRUE;
361+
}
362+
}
363+
else if (t1 == VIPS_TYPE_ARRAY_IMAGE) {
364+
int n1;
365+
void **array1 = (void **)
366+
vips_area_get_data(VIPS_AREA(p1), NULL, &n1, NULL, NULL);
367+
int n2;
368+
void **array2 = (void **)
369+
vips_area_get_data(VIPS_AREA(p2), NULL, &n2, NULL, NULL);
370+
371+
if (n1 != n2)
372+
return FALSE;
373+
else if (array1 == array2)
374+
return TRUE;
375+
else {
376+
for (int i = 0; i < n1; i++)
377+
if (array1[i] != array2[i])
378+
return FALSE;
379+
380+
return TRUE;
381+
}
382+
}
383+
else
384+
return p1 == p2;
385+
}
280386
if (generic == G_TYPE_PARAM_POINTER)
281387
return g_value_get_pointer(v1) == g_value_get_pointer(v2);
282388
if (generic == G_TYPE_PARAM_OBJECT)
@@ -815,6 +921,49 @@ vips_cache_trim(void)
815921
g_mutex_unlock(&vips_cache_lock);
816922
}
817923

924+
#ifdef DEBUG_LEAK
925+
static void *
926+
vips_cache_find_differences(VipsObject *object,
927+
GParamSpec *pspec,
928+
VipsArgumentClass *argument_class,
929+
VipsArgumentInstance *argument_instance,
930+
void *a, void *b)
931+
{
932+
VipsOperation *operation_before = VIPS_OPERATION(a);
933+
934+
if ((argument_class->flags & VIPS_ARGUMENT_CONSTRUCT) &&
935+
(argument_class->flags & VIPS_ARGUMENT_INPUT) &&
936+
argument_instance->assigned) {
937+
const char *name = g_param_spec_get_name(pspec);
938+
939+
GValue value_before = G_VALUE_INIT;
940+
g_object_get_property(G_OBJECT(operation_before), name, &value_before);
941+
unsigned int hash_before = vips_value_hash(pspec, &value_before);
942+
943+
GValue value_after = G_VALUE_INIT;
944+
g_object_get_property(G_OBJECT(object), name, &value_after);
945+
unsigned int hash_after = vips_value_hash(pspec, &value_after);
946+
947+
if (hash_before != hash_after) {
948+
g_warning("arg \"%s\" has changed value during build", name);
949+
950+
char *str_before = g_strdup_value_contents(&value_before);
951+
g_warning("\tvalue before: %s", str_before);
952+
g_free(str_before);
953+
954+
char *str_after = g_strdup_value_contents(&value_after);
955+
g_warning("\tvalue after: %s", str_after);
956+
g_free(str_after);
957+
}
958+
959+
g_value_unset(&value_before);
960+
g_value_unset(&value_after);
961+
}
962+
963+
return NULL;
964+
}
965+
#endif /*DEBUG_LEAK*/
966+
818967
/**
819968
* vips_cache_operation_buildp: (skip)
820969
* @operation: pointer to operation to lookup
@@ -901,6 +1050,11 @@ vips_cache_operation_buildp(VipsOperation **operation)
9011050
*/
9021051
if (vips__leak) {
9031052
hash_before = vips_operation_hash(*operation);
1053+
1054+
/* This isn't a deep copy, so it won't detect eg.
1055+
* operations modifying compound objects like VipsArrayInt.
1056+
* However, the hash_before value we save will.
1057+
*/
9041058
operation_before = vips_operation_copy(*operation);
9051059
}
9061060
#endif /*DEBUG_LEAK*/
@@ -912,25 +1066,14 @@ vips_cache_operation_buildp(VipsOperation **operation)
9121066
if (vips__leak &&
9131067
!(flags & VIPS_OPERATION_NOCACHE) &&
9141068
hash_before != vips_operation_hash(*operation)) {
915-
const char *name = (const char *)
916-
vips_argument_map(VIPS_OBJECT(*operation),
917-
vips_object_equal_arg, operation_before, NULL);
918-
VipsObject *object = VIPS_OBJECT(*operation);
919-
VipsObjectClass *class = VIPS_OBJECT_GET_CLASS(object);
920-
9211069
char txt[256];
9221070
VipsBuf buf = VIPS_BUF_STATIC(txt);
1071+
vips_object_summary(VIPS_OBJECT(operation_before), &buf);
1072+
g_warning("vips_cache_operation_buildp: "
1073+
"arg mismatch on build of: %s", vips_buf_all(&buf));
9231074

924-
VIPS_UNREF(operation_before);
925-
926-
vips_object_summary_class(class, &buf);
927-
vips_buf_appends(&buf, ", ");
928-
vips_object_summary(object, &buf);
929-
vips_buf_appends(&buf, ", ");
930-
vips_error(class->nickname, "arg \"%s\" changed during build, %s",
931-
name, vips_buf_all(&buf));
932-
933-
return -1;
1075+
vips_argument_map(VIPS_OBJECT(*operation),
1076+
vips_cache_find_differences, operation_before, NULL);
9341077
}
9351078

9361079
VIPS_UNREF(operation_before);

‎libvips/iofuncs/object.c‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -179,21 +179,21 @@
179179
* Input gobjects are automatically reffed, output gobjects automatically ref
180180
* us. We also automatically watch for "destroy" and unlink.
181181
*
182-
* [flags@Vips.ArgumentFlags.SET_ALWAYS] is handy for arguments which are set from C. For
183-
* example, [property@Image:width] is a property that gives access to the Xsize
184-
* member of struct _VipsImage. We default its 'assigned' to `TRUE`
185-
* since the field is always set directly by C.
182+
* [flags@Vips.ArgumentFlags.SET_ALWAYS] is handy for arguments which are set
183+
* from C. For example, [property@Image:width] is a property that gives
184+
* access to the Xsize member of struct _VipsImage. We default its
185+
* 'assigned' to `TRUE` since the field is always set directly by C.
186186
*
187-
* [flags@Vips.ArgumentFlags.DEPRECATED] arguments are not shown in help text, are not
188-
* looked for if required, are not checked for "have-been-set". You can
187+
* [flags@Vips.ArgumentFlags.DEPRECATED] arguments are not shown in help text,
188+
* are not looked for if required, are not checked for "have-been-set". You can
189189
* deprecate a required argument, but you must obviously add a new required
190190
* argument if you do.
191191
*
192-
* Input args with [flags@Vips.ArgumentFlags.MODIFY] will be modified by the operation.
193-
* This is used for things like the in-place drawing operations.
192+
* Input args with [flags@Vips.ArgumentFlags.MODIFY] will be modified by the
193+
* operation. This is used for things like the in-place drawing operations.
194194
*
195-
* [flags@Vips.ArgumentFlags.NON_HASHABLE] stops the argument being used in hash and
196-
* equality tests. It's useful for arguments like `revalidate` which
195+
* [flags@Vips.ArgumentFlags.NON_HASHABLE] stops the argument being used in
196+
* hash and equality tests. It's useful for arguments like `revalidate` which
197197
* control the behaviour of the operator cache.
198198
*/
199199

0 commit comments

Comments
 (0)