1 ///////////////////////////////////////////////////////////////////////////////
2 //
3 /// \file outqueue.c
4 /// \brief Output queue handling in multithreaded coding
5 //
6 // Author: Lasse Collin
7 //
8 // This file has been put into the public domain.
9 // You can do whatever you want with this file.
10 //
11 ///////////////////////////////////////////////////////////////////////////////
12
13 #include "outqueue.h"
14
15
16 /// This is to ease integer overflow checking: We may allocate up to
17 /// 2 * LZMA_THREADS_MAX buffers and we need some extra memory for other
18 /// data structures (that's the second /2).
19 #define BUF_SIZE_MAX (UINT64_MAX / LZMA_THREADS_MAX / 2 / 2)
20
21
22 static lzma_ret
get_options(uint64_t * bufs_alloc_size,uint32_t * bufs_count,uint64_t buf_size_max,uint32_t threads)23 get_options(uint64_t *bufs_alloc_size, uint32_t *bufs_count,
24 uint64_t buf_size_max, uint32_t threads)
25 {
26 if (threads > LZMA_THREADS_MAX || buf_size_max > BUF_SIZE_MAX)
27 return LZMA_OPTIONS_ERROR;
28
29 // The number of buffers is twice the number of threads.
30 // This wastes RAM but keeps the threads busy when buffers
31 // finish out of order.
32 //
33 // NOTE: If this is changed, update BUF_SIZE_MAX too.
34 *bufs_count = threads * 2;
35 *bufs_alloc_size = *bufs_count * buf_size_max;
36
37 return LZMA_OK;
38 }
39
40
41 extern uint64_t
lzma_outq_memusage(uint64_t buf_size_max,uint32_t threads)42 lzma_outq_memusage(uint64_t buf_size_max, uint32_t threads)
43 {
44 uint64_t bufs_alloc_size;
45 uint32_t bufs_count;
46
47 if (get_options(&bufs_alloc_size, &bufs_count, buf_size_max, threads)
48 != LZMA_OK)
49 return UINT64_MAX;
50
51 return sizeof(lzma_outq) + bufs_count * sizeof(lzma_outbuf)
52 + bufs_alloc_size;
53 }
54
55
56 extern lzma_ret
lzma_outq_init(lzma_outq * outq,const lzma_allocator * allocator,uint64_t buf_size_max,uint32_t threads)57 lzma_outq_init(lzma_outq *outq, const lzma_allocator *allocator,
58 uint64_t buf_size_max, uint32_t threads)
59 {
60 uint64_t bufs_alloc_size;
61 uint32_t bufs_count;
62
63 // Set bufs_count and bufs_alloc_size.
64 return_if_error(get_options(&bufs_alloc_size, &bufs_count,
65 buf_size_max, threads));
66
67 // Allocate memory if needed.
68 if (outq->buf_size_max != buf_size_max
69 || outq->bufs_allocated != bufs_count) {
70 lzma_outq_end(outq, allocator);
71
72 #if SIZE_MAX < UINT64_MAX
73 if (bufs_alloc_size > SIZE_MAX)
74 return LZMA_MEM_ERROR;
75 #endif
76
77 outq->bufs = lzma_alloc(bufs_count * sizeof(lzma_outbuf),
78 allocator);
79 outq->bufs_mem = lzma_alloc((size_t)(bufs_alloc_size),
80 allocator);
81
82 if (outq->bufs == NULL || outq->bufs_mem == NULL) {
83 lzma_outq_end(outq, allocator);
84 return LZMA_MEM_ERROR;
85 }
86 }
87
88 // Initialize the rest of the main structure. Initialization of
89 // outq->bufs[] is done when they are actually needed.
90 outq->buf_size_max = (size_t)(buf_size_max);
91 outq->bufs_allocated = bufs_count;
92 outq->bufs_pos = 0;
93 outq->bufs_used = 0;
94 outq->read_pos = 0;
95
96 return LZMA_OK;
97 }
98
99
100 extern void
lzma_outq_end(lzma_outq * outq,const lzma_allocator * allocator)101 lzma_outq_end(lzma_outq *outq, const lzma_allocator *allocator)
102 {
103 lzma_free(outq->bufs, allocator);
104 outq->bufs = NULL;
105
106 lzma_free(outq->bufs_mem, allocator);
107 outq->bufs_mem = NULL;
108
109 return;
110 }
111
112
113 extern lzma_outbuf *
lzma_outq_get_buf(lzma_outq * outq)114 lzma_outq_get_buf(lzma_outq *outq)
115 {
116 // Caller must have checked it with lzma_outq_has_buf().
117 assert(outq->bufs_used < outq->bufs_allocated);
118
119 // Initialize the new buffer.
120 lzma_outbuf *buf = &outq->bufs[outq->bufs_pos];
121 buf->buf = outq->bufs_mem + outq->bufs_pos * outq->buf_size_max;
122 buf->size = 0;
123 buf->finished = false;
124
125 // Update the queue state.
126 if (++outq->bufs_pos == outq->bufs_allocated)
127 outq->bufs_pos = 0;
128
129 ++outq->bufs_used;
130
131 return buf;
132 }
133
134
135 extern bool
lzma_outq_is_readable(const lzma_outq * outq)136 lzma_outq_is_readable(const lzma_outq *outq)
137 {
138 uint32_t i = outq->bufs_pos - outq->bufs_used;
139 if (outq->bufs_pos < outq->bufs_used)
140 i += outq->bufs_allocated;
141
142 return outq->bufs[i].finished;
143 }
144
145
146 extern lzma_ret
lzma_outq_read(lzma_outq * restrict outq,uint8_t * restrict out,size_t * restrict out_pos,size_t out_size,lzma_vli * restrict unpadded_size,lzma_vli * restrict uncompressed_size)147 lzma_outq_read(lzma_outq *restrict outq, uint8_t *restrict out,
148 size_t *restrict out_pos, size_t out_size,
149 lzma_vli *restrict unpadded_size,
150 lzma_vli *restrict uncompressed_size)
151 {
152 // There must be at least one buffer from which to read.
153 if (outq->bufs_used == 0)
154 return LZMA_OK;
155
156 // Get the buffer.
157 uint32_t i = outq->bufs_pos - outq->bufs_used;
158 if (outq->bufs_pos < outq->bufs_used)
159 i += outq->bufs_allocated;
160
161 lzma_outbuf *buf = &outq->bufs[i];
162
163 // If it isn't finished yet, we cannot read from it.
164 if (!buf->finished)
165 return LZMA_OK;
166
167 // Copy from the buffer to output.
168 lzma_bufcpy(buf->buf, &outq->read_pos, buf->size,
169 out, out_pos, out_size);
170
171 // Return if we didn't get all the data from the buffer.
172 if (outq->read_pos < buf->size)
173 return LZMA_OK;
174
175 // The buffer was finished. Tell the caller its size information.
176 *unpadded_size = buf->unpadded_size;
177 *uncompressed_size = buf->uncompressed_size;
178
179 // Free this buffer for further use.
180 --outq->bufs_used;
181 outq->read_pos = 0;
182
183 return LZMA_STREAM_END;
184 }
185