From 32f25d2f2095c13110640f27c99d0c95f2d49255 Mon Sep 17 00:00:00 2001
From: Tomas Farago <sensej007@email.cz>
Date: Wed, 1 Dec 2021 16:31:11 +0100
Subject: read: add image-start and image-step properties
to skip images in multi-image files. Fixes #164.
---
docs/generators.rst | 8 +++
src/readers/ufo-edf-reader.c | 31 ++++++---
src/readers/ufo-hdf5-reader.c | 13 +++-
src/readers/ufo-raw-reader.c | 16 ++++-
src/readers/ufo-reader.c | 10 +--
src/readers/ufo-reader.h | 12 ++--
src/readers/ufo-tiff-reader.c | 36 +++++++++--
src/ufo-read-task.c | 142 ++++++++++++++++++++++++++----------------
8 files changed, 190 insertions(+), 78 deletions(-)
diff --git a/docs/generators.rst b/docs/generators.rst
index 7e52fb0..9f063ab 100644
--- a/docs/generators.rst
+++ b/docs/generators.rst
@@ -34,10 +34,18 @@ File reader
First index from where files are read.
+ .. gobj:prop:: image-start:uint
+
+ First image index from where images are read in multi-image files.
+
.. gobj:prop:: step:uint
Number of files to skip.
+ .. gobj:prop:: image-step:uint
+
+ Number of images to skip in a multi-image file.
+
.. gobj:prop:: y:uint
Vertical coordinate from where to start reading.
diff --git a/src/readers/ufo-edf-reader.c b/src/readers/ufo-edf-reader.c
index cc52ef1..77d255b 100644
--- a/src/readers/ufo-edf-reader.c
+++ b/src/readers/ufo-edf-reader.c
@@ -26,6 +26,7 @@
struct _UfoEdfReaderPrivate {
FILE *fp;
+ guint start;
gssize size;
gsize height;
guint bytes_per_sample;
@@ -68,6 +69,7 @@ ufo_edf_reader_open (UfoReader *reader,
fseek (priv->fp, 0L, SEEK_END);
priv->size = (gsize) ftell (priv->fp);
fseek (priv->fp, 0L, SEEK_SET);
+ priv->start = start;
return TRUE;
}
@@ -93,19 +95,22 @@ ufo_edf_reader_data_available (UfoReader *reader)
return priv->fp != NULL && ftell (priv->fp) < priv->size;
}
-static void
+static gsize
ufo_edf_reader_read (UfoReader *reader,
UfoBuffer *buffer,
UfoRequisition *requisition,
guint roi_y,
guint roi_height,
- guint roi_step)
+ guint roi_step,
+ guint image_step)
{
UfoEdfReaderPrivate *priv;
gsize num_bytes;
gsize num_read;
gssize offset;
gchar *data;
+ gsize to_skip;
+ guint start = 0;
priv = UFO_EDF_READER_GET_PRIVATE (reader);
data = (gchar *) ufo_buffer_get_host_array (buffer, NULL);
@@ -113,12 +118,16 @@ ufo_edf_reader_read (UfoReader *reader,
/* size of the image width in bytes */
const gsize width = requisition->dims[0] * priv->bytes_per_sample;
const guint num_rows = requisition->dims[1];
+ if (priv->start) {
+ start = priv->start;
+ priv->start = 0;
+ }
const gsize end_position = ftell (priv->fp) + priv->height * width;
offset = 0;
- /* Go to the first desired row */
- fseek (priv->fp, roi_y * width, SEEK_CUR);
+ /* Go to the first desired row at *start* image index */
+ fseek (priv->fp, start * priv->height * width + roi_y * width, SEEK_CUR);
if (roi_step == 1) {
/* Read the full ROI at once if no stepping is specified */
@@ -126,14 +135,14 @@ ufo_edf_reader_read (UfoReader *reader,
num_read = fread (data, 1, num_bytes, priv->fp);
if (num_read != num_bytes)
- return;
+ return 0;
}
else {
for (guint i = 0; i < num_rows - 1; i++) {
num_read = fread (data + offset, 1, width, priv->fp);
if (num_read != width)
- return;
+ return 0;
offset += width;
fseek (priv->fp, (roi_step - 1) * width, SEEK_CUR);
@@ -144,12 +153,16 @@ ufo_edf_reader_read (UfoReader *reader,
num_read = fread (data + offset, 1, width, priv->fp);
if (num_read != width)
- return;
+ return 0;
}
/* Go to the image end to be in a consistent state for the next read */
fseek (priv->fp, end_position, SEEK_SET);
+ /* Skip the desired number of images */
+ to_skip = MIN (image_step - 1, (priv->size - (gsize) ftell (priv->fp)) / (priv->height * width));
+ fseek (priv->fp, to_skip * priv->height * width, SEEK_CUR);
+
if ((G_BYTE_ORDER == G_LITTLE_ENDIAN) && priv->big_endian) {
guint32 *conv = (guint32 *) ufo_buffer_get_host_array (buffer, NULL);
guint n_pixels = requisition->dims[0] * requisition->dims[1];
@@ -157,6 +170,8 @@ ufo_edf_reader_read (UfoReader *reader,
for (guint i = 0; i < n_pixels; i++)
conv[i] = g_ntohl (conv[i]);
}
+
+ return to_skip + 1;
}
static void
@@ -192,6 +207,7 @@ ufo_edf_reader_get_depth (const gchar *value, UfoBufferDepth *depth, guint *byte
static gboolean
ufo_edf_reader_get_meta (UfoReader *reader,
UfoRequisition *requisition,
+ gsize *num_images,
UfoBufferDepth *bitdepth,
GError **error)
{
@@ -270,6 +286,7 @@ ufo_edf_reader_get_meta (UfoReader *reader,
g_strfreev (key_value);
}
+ *num_images = requisition->dims[0] * requisition->dims[1] * priv->bytes_per_sample / (priv->size - data_position);
g_strfreev (tokens);
g_free (header);
return TRUE;
diff --git a/src/readers/ufo-hdf5-reader.c b/src/readers/ufo-hdf5-reader.c
index 039e520..e7c94df 100644
--- a/src/readers/ufo-hdf5-reader.c
+++ b/src/readers/ufo-hdf5-reader.c
@@ -116,18 +116,20 @@ ufo_hdf5_reader_data_available (UfoReader *reader)
return priv->current < priv->dims[0];
}
-static void
+static gsize
ufo_hdf5_reader_read (UfoReader *reader,
UfoBuffer *buffer,
UfoRequisition *requisition,
guint roi_y,
guint roi_height,
- guint roi_step)
+ guint roi_step,
+ guint image_step)
{
UfoHdf5ReaderPrivate *priv;
gpointer data;
hid_t dst_dataspace_id;
hsize_t dst_dims[2];
+ gsize num_read = 0;
priv = UFO_HDF5_READER_GET_PRIVATE (reader);
data = ufo_buffer_get_host_array (buffer, NULL);
@@ -143,12 +145,16 @@ ufo_hdf5_reader_read (UfoReader *reader,
H5Dread (priv->dataset_id, H5T_NATIVE_FLOAT, dst_dataspace_id, priv->src_dataspace_id, H5P_DEFAULT, data);
H5Sclose (dst_dataspace_id);
- priv->current++;
+ num_read = MIN (image_step, priv->dims[0] - priv->current);
+ priv->current += num_read;
+
+ return num_read;
}
static gboolean
ufo_hdf5_reader_get_meta (UfoReader *reader,
UfoRequisition *requisition,
+ gsize *num_images,
UfoBufferDepth *bitdepth,
GError **error)
{
@@ -159,6 +165,7 @@ ufo_hdf5_reader_get_meta (UfoReader *reader,
requisition->n_dims = 2;
requisition->dims[0] = priv->dims[2];
requisition->dims[1] = priv->dims[1];
+ *num_images = priv->dims[0];
*bitdepth = UFO_BUFFER_DEPTH_32F;
return TRUE;
}
diff --git a/src/readers/ufo-raw-reader.c b/src/readers/ufo-raw-reader.c
index d9b06b9..f57c0f8 100644
--- a/src/readers/ufo-raw-reader.c
+++ b/src/readers/ufo-raw-reader.c
@@ -123,19 +123,23 @@ ufo_raw_reader_data_available (UfoReader *reader)
return priv->fp != NULL && pos >= 0 && (((gulong) pos) + priv->pre_offset + priv->frame_size) <= priv->total_size;
}
-static void
+static gsize
ufo_raw_reader_read (UfoReader *reader,
UfoBuffer *buffer,
UfoRequisition *requisition,
guint roi_y,
guint roi_height,
- guint roi_step)
+ guint roi_step,
+ guint image_step)
{
UfoRawReaderPrivate *priv;
gchar *data;
+ gsize to_skip;
+ gsize page_size;
priv = UFO_RAW_READER_GET_PRIVATE (reader);
data = (gchar *) ufo_buffer_get_host_array (buffer, NULL);
+ page_size = priv->frame_size + priv->pre_offset + priv->post_offset;
fseek (priv->fp, priv->pre_offset, SEEK_CUR);
@@ -144,11 +148,18 @@ ufo_raw_reader_read (UfoReader *reader,
g_warning ("Could not read enough data");
fseek (priv->fp, priv->post_offset, SEEK_CUR);
+
+ /* Skip the desired number of images */
+ to_skip = MIN (image_step - 1, (priv->total_size - (gsize) ftell (priv->fp)) / page_size);
+ fseek (priv->fp, to_skip * page_size, SEEK_CUR);
+
+ return to_skip + 1;
}
static gboolean
ufo_raw_reader_get_meta (UfoReader *reader,
UfoRequisition *requisition,
+ gsize *num_images,
UfoBufferDepth *bitdepth,
GError **error)
{
@@ -158,6 +169,7 @@ ufo_raw_reader_get_meta (UfoReader *reader,
requisition->n_dims = 2;
requisition->dims[0] = priv->width;
requisition->dims[1] = priv->height;
+ *num_images = priv->total_size / priv->frame_size;
*bitdepth = priv->bitdepth;
return TRUE;
}
diff --git a/src/readers/ufo-reader.c b/src/readers/ufo-reader.c
index 499abf0..f831770 100644
--- a/src/readers/ufo-reader.c
+++ b/src/readers/ufo-reader.c
@@ -55,21 +55,23 @@ ufo_reader_data_available (UfoReader *reader)
gboolean
ufo_reader_get_meta (UfoReader *reader,
UfoRequisition *requisition,
+ gsize *num_images,
UfoBufferDepth *bitdepth,
GError **error)
{
- return UFO_READER_GET_IFACE (reader)->get_meta (reader, requisition, bitdepth, error);
+ return UFO_READER_GET_IFACE (reader)->get_meta (reader, requisition, num_images, bitdepth, error);
}
-void
+gsize
ufo_reader_read (UfoReader *reader,
UfoBuffer *buffer,
UfoRequisition *requisition,
guint roi_y,
guint roi_height,
- guint roi_step)
+ guint roi_step,
+ guint image_step)
{
- UFO_READER_GET_IFACE (reader)->read (reader, buffer, requisition, roi_y, roi_height, roi_step);
+ return UFO_READER_GET_IFACE (reader)->read (reader, buffer, requisition, roi_y, roi_height, roi_step, image_step);
}
static void
diff --git a/src/readers/ufo-reader.h b/src/readers/ufo-reader.h
index 52c6c4b..0752506 100644
--- a/src/readers/ufo-reader.h
+++ b/src/readers/ufo-reader.h
@@ -48,14 +48,16 @@ struct _UfoReaderIface {
gboolean (*data_available) (UfoReader *reader);
gboolean (*get_meta) (UfoReader *reader,
UfoRequisition *requisition,
+ gsize *num_images,
guint *bitdepth,
GError **error);
- void (*read) (UfoReader *reader,
+ gsize (*read) (UfoReader *reader,
UfoBuffer *buffer,
UfoRequisition *requisition,
guint roi_y,
guint roi_height,
- guint roi_step);
+ guint roi_step,
+ guint image_step);
};
gboolean ufo_reader_can_open (UfoReader *reader,
@@ -68,14 +70,16 @@ void ufo_reader_close (UfoReader *reader);
gboolean ufo_reader_data_available (UfoReader *reader);
gboolean ufo_reader_get_meta (UfoReader *reader,
UfoRequisition *requisition,
+ gsize *num_images,
UfoBufferDepth *bitdepth,
GError **error);
-void ufo_reader_read (UfoReader *reader,
+gsize ufo_reader_read (UfoReader *reader,
UfoBuffer *buffer,
UfoRequisition *requisition,
guint roi_y,
guint roi_height,
- guint roi_step);
+ guint roi_step,
+ guint image_step);
GType ufo_reader_get_type (void);
diff --git a/src/readers/ufo-tiff-reader.c b/src/readers/ufo-tiff-reader.c
index 0bac745..678a97b 100644
--- a/src/readers/ufo-tiff-reader.c
+++ b/src/readers/ufo-tiff-reader.c
@@ -26,6 +26,7 @@
struct _UfoTiffReaderPrivate {
TIFF *tiff;
gboolean more;
+ gsize num_images;
};
static void ufo_reader_interface_init (UfoReaderIface *iface);
@@ -59,8 +60,9 @@ ufo_tiff_reader_open (UfoReader *reader,
UfoTiffReaderPrivate *priv;
priv = UFO_TIFF_READER_GET_PRIVATE (reader);
+ priv->num_images = 0;
priv->tiff = TIFFOpen (filename, "r");
- priv->more = TRUE;
+ priv->more = FALSE;
if (priv->tiff == NULL) {
g_set_error (error, UFO_TASK_ERROR, UFO_TASK_ERROR_SETUP,
@@ -68,8 +70,18 @@ ufo_tiff_reader_open (UfoReader *reader,
return FALSE;
}
- for (guint i = 0; i < start; i++)
- priv->more = TIFFReadDirectory (priv->tiff) == 1;
+ do {
+ priv->num_images++;
+ } while (TIFFReadDirectory(priv->tiff));
+
+ if (start < priv->num_images) {
+ priv->more = TRUE;
+ if (TIFFSetDirectory (priv->tiff, start) != 1) {
+ g_set_error (error, UFO_TASK_ERROR, UFO_TASK_ERROR_SETUP,
+ "Cannot find first image in %s", filename);
+ return FALSE;
+ }
+ }
return TRUE;
}
@@ -172,16 +184,18 @@ read_64_bit_data (UfoTiffReaderPrivate *priv,
g_free (src);
}
-static void
+static gsize
ufo_tiff_reader_read (UfoReader *reader,
UfoBuffer *buffer,
UfoRequisition *requisition,
guint roi_y,
guint roi_height,
- guint roi_step)
+ guint roi_step,
+ guint image_step)
{
UfoTiffReaderPrivate *priv;
guint16 bits;
+ gsize num_read = 0;
priv = UFO_TIFF_READER_GET_PRIVATE (reader);
@@ -192,12 +206,21 @@ ufo_tiff_reader_read (UfoReader *reader,
else
read_data (priv, buffer, requisition, bits, roi_y, roi_height, roi_step);
- priv->more = TIFFReadDirectory (priv->tiff) == 1;
+ do {
+ priv->more = TIFFReadDirectory (priv->tiff) == 1;
+ num_read++;
+ if (!priv->more) {
+ break;
+ }
+ } while (num_read < image_step);
+
+ return num_read;
}
static gboolean
ufo_tiff_reader_get_meta (UfoReader *reader,
UfoRequisition *requisition,
+ gsize *num_images,
UfoBufferDepth *bitdepth,
GError **error)
{
@@ -219,6 +242,7 @@ ufo_tiff_reader_get_meta (UfoReader *reader,
requisition->dims[0] = (gsize) width;
requisition->dims[1] = (gsize) height;
requisition->dims[2] = samples == 3 ? 3 : 0;
+ *num_images = priv->num_images;
switch (bits_per_sample) {
case 8:
diff --git a/src/ufo-read-task.c b/src/ufo-read-task.c
index 1587031..b2a0457 100644
--- a/src/ufo-read-task.c
+++ b/src/ufo-read-task.c
@@ -70,6 +70,7 @@ struct _UfoReadTaskPrivate {
guint current;
guint step;
guint start;
+ guint image_start;
guint number;
guint retries;
guint retry_timeout;
@@ -82,6 +83,7 @@ struct _UfoReadTaskPrivate {
guint roi_y;
guint roi_height;
guint roi_step;
+ guint image_step;
UfoReader *reader;
UfoEdfReader *edf_reader;
@@ -110,11 +112,13 @@ enum {
PROP_0,
PROP_PATH,
PROP_START,
+ PROP_IMAGE_START,
PROP_NUMBER,
PROP_STEP,
PROP_ROI_Y,
PROP_ROI_HEIGHT,
PROP_ROI_STEP,
+ PROP_IMAGE_STEP,
PROP_CONVERT,
PROP_RAW_WIDTH,
PROP_RAW_HEIGHT,
@@ -224,7 +228,6 @@ ufo_read_task_setup (UfoTask *task,
return;
}
- priv->start = 0;
priv->current = 0;
}
@@ -258,69 +261,73 @@ ufo_read_task_get_requisition (UfoTask *task,
{
UfoReadTaskPrivate *priv;
const gchar *filename;
+ gsize num_images = 0;
priv = UFO_READ_TASK_GET_PRIVATE (UFO_READ_TASK (task));
- if (priv->reader == NULL) {
- filename = (gchar *) priv->current_element->data;
- priv->reader = get_reader (priv, filename);
-
- if (!ufo_reader_open (priv->reader, filename, priv->start, error))
- return;
-
- priv->start = 0;
- }
-
- if (!ufo_reader_data_available (priv->reader)) {
+ if (!priv->reader || !ufo_reader_data_available (priv->reader)) {
GList *last_element;
guint tries;
- ufo_reader_close (priv->reader);
- last_element = priv->current_element;
- priv->current_element = g_list_nth (priv->current_element, priv->step);
-
- if (priv->current_element == NULL) {
- if (priv->retries == 0 || priv->current == priv->number) {
- priv->done = TRUE;
- priv->reader = NULL;
- return;
+ while (TRUE) {
+ /* Keep skipping files until we find one with enough images to start
+ * reading at priv->image_start index. */
+ if (priv->reader) {
+ ufo_reader_close (priv->reader);
+ last_element = priv->current_element;
+ priv->current_element = g_list_nth (priv->current_element, priv->step);
}
- for (tries = 0; tries < priv->retries && priv->current_element == NULL; tries++) {
- GList *new_list;
- GList *match;
-
- g_debug ("read: retry %i/%i, waiting %is for new files", tries + 1, priv->retries, priv->retry_timeout);
- g_usleep (priv->retry_timeout * G_USEC_PER_SEC);
- new_list = g_list_sort (read_filenames (priv), (GCompareFunc) g_strcmp0);
- match = g_list_find_custom (new_list, last_element->data, (GCompareFunc) g_strcmp0);
-
- if (match != g_list_last (new_list)) {
- g_list_free_full (priv->filenames, (GDestroyNotify) g_free);
- priv->filenames = new_list;
- priv->current_element = g_list_next (match);
- }
- else {
- g_list_free_full (new_list, (GDestroyNotify) g_free);
- }
+ if (priv->current_element == NULL) {
+ if (priv->retries == 0 || priv->current == priv->number) {
+ priv->done = TRUE;
+ priv->reader = NULL;
+ return;
+ }
+
+ for (tries = 0; tries < priv->retries && priv->current_element == NULL; tries++) {
+ GList *new_list;
+ GList *match;
+
+ g_debug ("read: retry %i/%i, waiting %is for new files", tries + 1, priv->retries, priv->retry_timeout);
+ g_usleep (priv->retry_timeout * G_USEC_PER_SEC);
+ new_list = g_list_sort (read_filenames (priv), (GCompareFunc) g_strcmp0);
+ match = g_list_find_custom (new_list, last_element->data, (GCompareFunc) g_strcmp0);
+
+ if (match != g_list_last (new_list)) {
+ g_list_free_full (priv->filenames, (GDestroyNotify) g_free);
+ priv->filenames = new_list;
+ priv->current_element = g_list_next (match);
+ }
+ else {
+ g_list_free_full (new_list, (GDestroyNotify) g_free);
+ }
+ }
+
+ if (priv->current_element == NULL) {
+ priv->done = TRUE;
+ priv->reader = NULL;
+ return;
+ }
}
- if (priv->current_element == NULL) {
- priv->done = TRUE;
- priv->reader = NULL;
+ filename = (gchar *) priv->current_element->data;
+ priv->reader = get_reader (priv, filename);
+
+ if (!ufo_reader_open (priv->reader, filename, priv->image_start, error))
return;
+ if (!ufo_reader_get_meta (priv->reader, requisition, &num_images, &priv->depth, error))
+ return;
+
+ if (priv->image_start >= num_images) {
+ priv->image_start -= num_images;
+ } else {
+ priv->image_start = 0;
+ break;
}
}
-
- filename = (gchar *) priv->current_element->data;
- priv->reader = get_reader (priv, filename);
-
- if (!ufo_reader_open (priv->reader, filename, 0, error))
- return;
}
- if (!ufo_reader_get_meta (priv->reader, requisition, &priv->depth, error))
- return;
if (priv->depth > 32)
/*
@@ -376,13 +383,16 @@ ufo_read_task_generate (UfoTask *task,
UfoRequisition *requisition)
{
UfoReadTaskPrivate *priv;
+ guint num_processed;
priv = UFO_READ_TASK_GET_PRIVATE (UFO_READ_TASK (task));
if (priv->current == priv->number || priv->done)
return FALSE;
- ufo_reader_read (priv->reader, output, requisition, priv->roi_y, priv->roi_height, priv->roi_step);
+ num_processed = ufo_reader_read (priv->reader, output, requisition, priv->roi_y, priv->roi_height,
+ priv->roi_step, priv->image_step);
+ priv->image_start = priv->image_step - num_processed;
if ((priv->depth != UFO_BUFFER_DEPTH_32F) && priv->convert)
ufo_buffer_convert (output, priv->depth);
@@ -416,12 +426,18 @@ ufo_read_task_set_property (GObject *object,
case PROP_ROI_STEP:
priv->roi_step = g_value_get_uint (value);
break;
+ case PROP_IMAGE_STEP:
+ priv->image_step = g_value_get_uint (value);
+ break;
case PROP_CONVERT:
priv->convert = g_value_get_boolean (value);
break;
case PROP_START:
priv->start = g_value_get_uint (value);
break;
+ case PROP_IMAGE_START:
+ priv->image_start = g_value_get_uint (value);
+ break;
case PROP_NUMBER:
priv->number = g_value_get_uint (value);
break;
@@ -479,12 +495,18 @@ ufo_read_task_get_property (GObject *object,
case PROP_ROI_STEP:
g_value_set_uint (value, priv->roi_step);
break;
+ case PROP_IMAGE_STEP:
+ g_value_set_uint (value, priv->image_step);
+ break;
case PROP_CONVERT:
g_value_set_boolean (value, priv->convert);
break;
case PROP_START:
g_value_set_uint (value, priv->start);
break;
+ case PROP_IMAGE_START:
+ g_value_set_uint (value, priv->image_start);
+ break;
case PROP_NUMBER:
g_value_set_uint (value, priv->number);
break;
@@ -613,6 +635,13 @@ ufo_read_task_class_init(UfoReadTaskClass *klass)
1, G_MAXUINT, 1,
G_PARAM_READWRITE);
+ properties[PROP_IMAGE_STEP] =
+ g_param_spec_uint ("image-step",
+ "Read every \"step\" image",
+ "Read every \"step\" image",
+ 1, G_MAXUINT, 1,
+ G_PARAM_READWRITE);
+
properties[PROP_CONVERT] =
g_param_spec_boolean ("convert",
"Enable automatic conversion",
@@ -627,10 +656,17 @@ ufo_read_task_class_init(UfoReadTaskClass *klass)
0, G_MAXUINT, 0,
G_PARAM_READWRITE);
+ properties[PROP_IMAGE_START] =
+ g_param_spec_uint ("image-start",
+ "Offset to the first read image in a multi-image file",
+ "Offset to the first read image in a multi-image file",
+ 0, G_MAXUINT, 0,
+ G_PARAM_READWRITE);
+
properties[PROP_NUMBER] =
g_param_spec_uint ("number",
- "Number of files that will be read at most",
- "Number of files that will be read at most",
+ "Number of images that will be read at most",
+ "Number of images that will be read at most",
0, G_MAXUINT, G_MAXUINT,
G_PARAM_READWRITE);
@@ -708,8 +744,10 @@ ufo_read_task_init(UfoReadTask *self)
priv->roi_y = 0;
priv->roi_height = 0;
priv->roi_step = 1;
+ priv->image_step = 1;
priv->convert = TRUE;
priv->start = 0;
+ priv->image_start = 0;
priv->number = G_MAXUINT;
priv->retries = 0;
priv->retry_timeout = 1;
--
cgit v1.2.3
From 9d37808d929b69d1526362ddc17f60774c08ac54 Mon Sep 17 00:00:00 2001
From: Tomas Farago <sensej007@email.cz>
Date: Wed, 1 Dec 2021 16:31:45 +0100
Subject: Test multi-image readers
---
tests/CMakeLists.txt | 9 ++++++
tests/check-multipage-readers | 24 ++++++++++++++++
tests/make-input-multipage-readers | 18 ++++++++++++
tests/meson.build | 11 ++++++-
tests/test-multipage-readers.sh | 59 ++++++++++++++++++++++++++++++++++++++
5 files changed, 120 insertions(+), 1 deletion(-)
create mode 100755 tests/check-multipage-readers
create mode 100755 tests/make-input-multipage-readers
create mode 100755 tests/test-multipage-readers.sh
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 8de94f1..d2edac1 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -23,3 +23,12 @@ add_test(test_core_149
add_test(test_nlm
${BASH} "${CMAKE_CURRENT_SOURCE_DIR}/test-nlm.sh")
+
+add_test(test_multipage_readers
+ ${BASH} "${CMAKE_CURRENT_SOURCE_DIR}/test-multipage-readers.sh")
+
+file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/make-input-multipage-readers
+ DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tests)
+
+file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/check-multipage-readers
+ DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tests)
diff --git a/tests/check-multipage-readers b/tests/check-multipage-readers
new file mode 100755
index 0000000..4e1bd9e
--- /dev/null
+++ b/tests/check-multipage-readers
@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+import sys
+import numpy as np
+import tifffile
+
+
+def main(gt):
+ im = tifffile.imread('multipage-out.tif')
+ if im.ndim == 2:
+ im = im[np.newaxis]
+
+ collected = []
+ for i in range(im.shape[0]):
+ collected.append(int(im[i, 0, 0]))
+ if int(gt[i]) != collected[i]:
+ print('Sequences do not match', gt[:i + 1], collected[:i + 1])
+ return 1
+
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[1:]))
diff --git a/tests/make-input-multipage-readers b/tests/make-input-multipage-readers
new file mode 100755
index 0000000..a565122
--- /dev/null
+++ b/tests/make-input-multipage-readers
@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+import h5py
+import numpy as np
+import tifffile
+
+
+base = np.ones((15, 128, 128), dtype=np.uint16) * np.arange(15, dtype=np.uint16)[:, np.newaxis, np.newaxis]
+fmt = 'multipage-image-{:>02}.{}'
+
+
+for i in range(4):
+ current = base + i * len(base)
+ tifffile.imsave(fmt.format(i, 'tif'), current)
+ with h5py.File(fmt.format(i, 'h5'), 'w') as f:
+ dset = f.create_dataset('images', data=current)
+ with open(fmt.format(i, 'raw'), 'wb') as f:
+ f.write(current.tostring())
diff --git a/tests/meson.build b/tests/meson.build
index 6b720d2..4ab4156 100644
--- a/tests/meson.build
+++ b/tests/meson.build
@@ -4,7 +4,8 @@ tests = [
'test-161',
'test-core-149',
'test-file-write-regression',
- 'test-nlm'
+ 'test-nlm',
+ 'test-multipage-readers'
]
tiffinfo = find_program('tiffinfo', required : false)
@@ -17,6 +18,14 @@ test_env = [
'UFO_PLUGIN_PATH=@0@'.format(join_paths(meson.build_root(), 'src'))
]
+configure_file(input: 'make-input-multipage-readers',
+ output: 'make-input-multipage-readers',
+ copy: true)
+
+configure_file(input: 'check-multipage-readers',
+ output: 'check-multipage-readers',
+ copy: true)
+
foreach t: tests
test(t, find_program('@0@.sh'.format(t)), env: test_env)
endforeach
diff --git a/tests/test-multipage-readers.sh b/tests/test-multipage-readers.sh
new file mode 100755
index 0000000..ab5845b
--- /dev/null
+++ b/tests/test-multipage-readers.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+
+tests/make-input-multipage-readers
+
+# TIFF
+ufo-launch -q read path=multipage-image*.tif image-start=5 image-step=8 ! write filename=multipage-out.tif tiff-bigtiff=False
+tests/check-multipage-readers 5 13 21 29 37 45 53
+if [ $? -ne 0 ]; then
+ exit 1;
+fi
+ufo-launch -q read path=multipage-image*.tif image-start=17 image-step=8 ! write filename=multipage-out.tif tiff-bigtiff=False
+tests/check-multipage-readers 17 25 33 41 49 57
+if [ $? -ne 0 ]; then
+ exit 1;
+fi
+ufo-launch -q read path=multipage-image*.tif image-start=5 image-step=30 ! write filename=multipage-out.tif tiff-bigtiff=False
+tests/check-multipage-readers 5 35
+if [ $? -ne 0 ]; then
+ exit 1;
+fi
+ufo-launch -q read path=multipage-image*.tif image-start=17 image-step=30 ! write filename=multipage-out.tif tiff-bigtiff=False
+tests/check-multipage-readers 17 47
+if [ $? -ne 0 ]; then
+ exit 1;
+fi
+
+# RAW
+ufo-launch -q read raw-width=128 raw-height=128 raw-bitdepth=16 path=multipage-image*.raw image-start=5 image-step=8 ! write filename=multipage-out.tif tiff-bigtiff=False
+tests/check-multipage-readers 5 13 21 29 37 45 53
+if [ $? -ne 0 ]; then
+ exit 1;
+fi
+ufo-launch -q read raw-width=128 raw-height=128 raw-bitdepth=16 path=multipage-image*.raw image-start=17 image-step=8 ! write filename=multipage-out.tif tiff-bigtiff=False
+tests/check-multipage-readers 17 25 33 41 49 57
+if [ $? -ne 0 ]; then
+ exit 1;
+fi
+ufo-launch -q read raw-width=128 raw-height=128 raw-bitdepth=16 path=multipage-image*.raw image-start=5 image-step=30 ! write filename=multipage-out.tif tiff-bigtiff=False
+tests/check-multipage-readers 5 35
+if [ $? -ne 0 ]; then
+ exit 1;
+fi
+ufo-launch -q read raw-width=128 raw-height=128 raw-bitdepth=16 path=multipage-image*.raw image-start=17 image-step=30 ! write filename=multipage-out.tif tiff-bigtiff=False
+tests/check-multipage-readers 17 47
+if [ $? -ne 0 ]; then
+ exit 1;
+fi
+
+# HDF5 (no multiple files reading support, so just work with one)
+ufo-launch -q read path=multipage-image-00.h5:/images image-start=5 image-step=8 ! write filename=multipage-out.tif tiff-bigtiff=False
+tests/check-multipage-readers 5 13
+if [ $? -ne 0 ]; then
+ exit 1;
+fi
+ufo-launch -q read path=multipage-image-00.h5:/images image-start=5 image-step=80 ! write filename=multipage-out.tif tiff-bigtiff=False
+tests/check-multipage-readers 5
+if [ $? -ne 0 ]; then
+ exit 1;
+fi
--
cgit v1.2.3
From a75fe709b9d3066af22c3d54c8fb1066afd7e4cc Mon Sep 17 00:00:00 2001
From: Tomas Farago <sensej007@email.cz>
Date: Fri, 3 Dec 2021 14:34:23 +0100
Subject: read: remove retries
---
src/ufo-read-task.c | 75 +++--------------------------------------------------
1 file changed, 3 insertions(+), 72 deletions(-)
diff --git a/src/ufo-read-task.c b/src/ufo-read-task.c
index b2a0457..f4818e0 100644
--- a/src/ufo-read-task.c
+++ b/src/ufo-read-task.c
@@ -72,8 +72,6 @@ struct _UfoReadTaskPrivate {
guint start;
guint image_start;
guint number;
- guint retries;
- guint retry_timeout;
gboolean done;
gboolean single;
@@ -126,8 +124,6 @@ enum {
PROP_RAW_PRE_OFFSET,
PROP_RAW_POST_OFFSET,
PROP_TYPE,
- PROP_RETRIES,
- PROP_RETRY_TIMEOUT,
N_PROPERTIES
};
@@ -222,12 +218,6 @@ ufo_read_task_setup (UfoTask *task,
return;
}
- if (priv->number == G_MAXUINT && priv->retries > 0) {
- g_set_error (error, UFO_TASK_ERROR, UFO_TASK_ERROR_SETUP,
- "`retries' but not `number' set");
- return;
- }
-
priv->current = 0;
}
@@ -266,49 +256,18 @@ ufo_read_task_get_requisition (UfoTask *task,
priv = UFO_READ_TASK_GET_PRIVATE (UFO_READ_TASK (task));
if (!priv->reader || !ufo_reader_data_available (priv->reader)) {
- GList *last_element;
- guint tries;
-
while (TRUE) {
/* Keep skipping files until we find one with enough images to start
* reading at priv->image_start index. */
if (priv->reader) {
ufo_reader_close (priv->reader);
- last_element = priv->current_element;
priv->current_element = g_list_nth (priv->current_element, priv->step);
}
if (priv->current_element == NULL) {
- if (priv->retries == 0 || priv->current == priv->number) {
- priv->done = TRUE;
- priv->reader = NULL;
- return;
- }
-
- for (tries = 0; tries < priv->retries && priv->current_element == NULL; tries++) {
- GList *new_list;
- GList *match;
-
- g_debug ("read: retry %i/%i, waiting %is for new files", tries + 1, priv->retries, priv->retry_timeout);
- g_usleep (priv->retry_timeout * G_USEC_PER_SEC);
- new_list = g_list_sort (read_filenames (priv), (GCompareFunc) g_strcmp0);
- match = g_list_find_custom (new_list, last_element->data, (GCompareFunc) g_strcmp0);
-
- if (match != g_list_last (new_list)) {
- g_list_free_full (priv->filenames, (GDestroyNotify) g_free);
- priv->filenames = new_list;
- priv->current_element = g_list_next (match);
- }
- else {
- g_list_free_full (new_list, (GDestroyNotify) g_free);
- }
- }
-
- if (priv->current_element == NULL) {
- priv->done = TRUE;
- priv->reader = NULL;
- return;
- }
+ priv->done = TRUE;
+ priv->reader = NULL;
+ return;
}
filename = (gchar *) priv->current_element->data;
@@ -459,12 +418,6 @@ ufo_read_task_set_property (GObject *object,
case PROP_TYPE:
priv->type = g_value_get_enum (value);
break;
- case PROP_RETRIES:
- priv->retries = g_value_get_uint (value);
- break;
- case PROP_RETRY_TIMEOUT:
- priv->retry_timeout = g_value_get_uint (value);
- break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
break;
@@ -528,12 +481,6 @@ ufo_read_task_get_property (GObject *object,
case PROP_TYPE:
g_value_set_enum (value, priv->type);
break;
- case PROP_RETRIES:
- g_value_set_uint (value, priv->retries);
- break;
- case PROP_RETRY_TIMEOUT:
- g_value_set_uint (value, priv->retry_timeout);
- break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
break;
@@ -713,20 +660,6 @@ ufo_read_task_class_init(UfoReadTaskClass *klass)
TYPE_UNSPECIFIED,
G_PARAM_READWRITE);
- properties[PROP_RETRIES] =
- g_param_spec_uint ("retries",
- "Number of read retries",
- "Number of read retries",
- 0, G_MAXUINT, 0,
- G_PARAM_READWRITE);
-
- properties[PROP_RETRY_TIMEOUT] =
- g_param_spec_uint ("retry-timeout",
- "Time in seconds to wait between retries",
- "Time in seconds to wait between retries",
- 0, G_MAXUINT, 1,
- G_PARAM_READWRITE);
-
for (guint i = PROP_0 + 1; i < N_PROPERTIES; i++)
g_object_class_install_property (gobject_class, i, properties[i]);
@@ -749,8 +682,6 @@ ufo_read_task_init(UfoReadTask *self)
priv->start = 0;
priv->image_start = 0;
priv->number = G_MAXUINT;
- priv->retries = 0;
- priv->retry_timeout = 1;
priv->depth = UFO_BUFFER_DEPTH_32F;
priv->edf_reader = ufo_edf_reader_new ();
--
cgit v1.2.3
From 9f49adb9b922cecc6cdabb437701a2005953b265 Mon Sep 17 00:00:00 2001
From: Tomas Farago <sensej007@email.cz>
Date: Fri, 3 Dec 2021 14:36:46 +0100
Subject: docs: remove retries and retry-timeout
---
docs/generators.rst | 12 ------------
1 file changed, 12 deletions(-)
diff --git a/docs/generators.rst b/docs/generators.rst
index 9f063ab..d0925f5 100644
--- a/docs/generators.rst
+++ b/docs/generators.rst
@@ -88,18 +88,6 @@ File reader
example, to load `.foo` files as raw files, set the ``type`` property to
`raw`.
- .. gobj:prop:: retries:uint
-
- Set the number of retries in case files do not exist yet and are being
- written. If you set this, you *must* also set ``number`` otherwise you
- would have to wait basically forever for the execution to finish. Note,
- that only files are considered which come after the last successful
- filename.
-
- .. gobj:prop:: retry-timeout:uint
-
- Seconds to wait before reading new files.
-
Memory reader
=============
--
cgit v1.2.3