encode.h 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501
  1. /* Copyright 2013 Google Inc. All Rights Reserved.
  2. Distributed under MIT license.
  3. See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
  4. */
  5. /**
  6. * @file
  7. * API for Brotli compression.
  8. */
  9. #ifndef BROTLI_ENC_ENCODE_H_
  10. #define BROTLI_ENC_ENCODE_H_
  11. #include <brotli/port.h>
  12. #include <brotli/shared_dictionary.h>
  13. #include <brotli/types.h>
  14. #if defined(__cplusplus) || defined(c_plusplus)
  15. extern "C" {
  16. #endif
  17. /** Minimal value for ::BROTLI_PARAM_LGWIN parameter. */
  18. #define BROTLI_MIN_WINDOW_BITS 10
  19. /**
  20. * Maximal value for ::BROTLI_PARAM_LGWIN parameter.
  21. *
  22. * @note equal to @c BROTLI_MAX_DISTANCE_BITS constant.
  23. */
  24. #define BROTLI_MAX_WINDOW_BITS 24
  25. /**
  26. * Maximal value for ::BROTLI_PARAM_LGWIN parameter
  27. * in "Large Window Brotli" (32-bit).
  28. */
  29. #define BROTLI_LARGE_MAX_WINDOW_BITS 30
  30. /** Minimal value for ::BROTLI_PARAM_LGBLOCK parameter. */
  31. #define BROTLI_MIN_INPUT_BLOCK_BITS 16
  32. /** Maximal value for ::BROTLI_PARAM_LGBLOCK parameter. */
  33. #define BROTLI_MAX_INPUT_BLOCK_BITS 24
  34. /** Minimal value for ::BROTLI_PARAM_QUALITY parameter. */
  35. #define BROTLI_MIN_QUALITY 0
  36. /** Maximal value for ::BROTLI_PARAM_QUALITY parameter. */
  37. #define BROTLI_MAX_QUALITY 11
  38. /** Options for ::BROTLI_PARAM_MODE parameter. */
  39. typedef enum BrotliEncoderMode {
  40. /**
  41. * Default compression mode.
  42. *
  43. * In this mode compressor does not know anything in advance about the
  44. * properties of the input.
  45. */
  46. BROTLI_MODE_GENERIC = 0,
  47. /** Compression mode for UTF-8 formatted text input. */
  48. BROTLI_MODE_TEXT = 1,
  49. /** Compression mode used in WOFF 2.0. */
  50. BROTLI_MODE_FONT = 2
  51. } BrotliEncoderMode;
  52. /** Default value for ::BROTLI_PARAM_QUALITY parameter. */
  53. #define BROTLI_DEFAULT_QUALITY 11
  54. /** Default value for ::BROTLI_PARAM_LGWIN parameter. */
  55. #define BROTLI_DEFAULT_WINDOW 22
  56. /** Default value for ::BROTLI_PARAM_MODE parameter. */
  57. #define BROTLI_DEFAULT_MODE BROTLI_MODE_GENERIC
  58. /** Operations that can be performed by streaming encoder. */
  59. typedef enum BrotliEncoderOperation {
  60. /**
  61. * Process input.
  62. *
  63. * Encoder may postpone producing output, until it has processed enough input.
  64. */
  65. BROTLI_OPERATION_PROCESS = 0,
  66. /**
  67. * Produce output for all processed input.
  68. *
  69. * Actual flush is performed when input stream is depleted and there is enough
  70. * space in output stream. This means that client should repeat
  71. * ::BROTLI_OPERATION_FLUSH operation until @p available_in becomes @c 0, and
  72. * ::BrotliEncoderHasMoreOutput returns ::BROTLI_FALSE. If output is acquired
  73. * via ::BrotliEncoderTakeOutput, then operation should be repeated after
  74. * output buffer is drained.
  75. *
  76. * @warning Until flush is complete, client @b SHOULD @b NOT swap,
  77. * reduce or extend input stream.
  78. *
  79. * When flush is complete, output data will be sufficient for decoder to
  80. * reproduce all the given input.
  81. */
  82. BROTLI_OPERATION_FLUSH = 1,
  83. /**
  84. * Finalize the stream.
  85. *
  86. * Actual finalization is performed when input stream is depleted and there is
  87. * enough space in output stream. This means that client should repeat
  88. * ::BROTLI_OPERATION_FINISH operation until @p available_in becomes @c 0, and
  89. * ::BrotliEncoderHasMoreOutput returns ::BROTLI_FALSE. If output is acquired
  90. * via ::BrotliEncoderTakeOutput, then operation should be repeated after
  91. * output buffer is drained.
  92. *
  93. * @warning Until finalization is complete, client @b SHOULD @b NOT swap,
  94. * reduce or extend input stream.
  95. *
  96. * Helper function ::BrotliEncoderIsFinished checks if stream is finalized and
  97. * output fully dumped.
  98. *
  99. * Adding more input data to finalized stream is impossible.
  100. */
  101. BROTLI_OPERATION_FINISH = 2,
  102. /**
  103. * Emit metadata block to stream.
  104. *
  105. * Metadata is opaque to Brotli: neither encoder, nor decoder processes this
  106. * data or relies on it. It may be used to pass some extra information from
  107. * encoder client to decoder client without interfering with main data stream.
  108. *
  109. * @note Encoder may emit empty metadata blocks internally, to pad encoded
  110. * stream to byte boundary.
  111. *
  112. * @warning Until emitting metadata is complete client @b SHOULD @b NOT swap,
  113. * reduce or extend input stream.
  114. *
  115. * @warning The whole content of input buffer is considered to be the content
  116. * of metadata block. Do @b NOT @e append metadata to input stream,
  117. * before it is depleted with other operations.
  118. *
  119. * Stream is soft-flushed before metadata block is emitted. Metadata block
  120. * @b MUST be no longer than than 16MiB.
  121. */
  122. BROTLI_OPERATION_EMIT_METADATA = 3
  123. } BrotliEncoderOperation;
  124. /** Options to be used with ::BrotliEncoderSetParameter. */
  125. typedef enum BrotliEncoderParameter {
  126. /**
  127. * Tune encoder for specific input.
  128. *
  129. * ::BrotliEncoderMode enumerates all available values.
  130. */
  131. BROTLI_PARAM_MODE = 0,
  132. /**
  133. * The main compression speed-density lever.
  134. *
  135. * The higher the quality, the slower the compression. Range is
  136. * from ::BROTLI_MIN_QUALITY to ::BROTLI_MAX_QUALITY.
  137. */
  138. BROTLI_PARAM_QUALITY = 1,
  139. /**
  140. * Recommended sliding LZ77 window size.
  141. *
  142. * Encoder may reduce this value, e.g. if input is much smaller than
  143. * window size.
  144. *
  145. * Window size is `(1 << value) - 16`.
  146. *
  147. * Range is from ::BROTLI_MIN_WINDOW_BITS to ::BROTLI_MAX_WINDOW_BITS.
  148. */
  149. BROTLI_PARAM_LGWIN = 2,
  150. /**
  151. * Recommended input block size.
  152. *
  153. * Encoder may reduce this value, e.g. if input is much smaller than input
  154. * block size.
  155. *
  156. * Range is from ::BROTLI_MIN_INPUT_BLOCK_BITS to
  157. * ::BROTLI_MAX_INPUT_BLOCK_BITS.
  158. *
  159. * @note Bigger input block size allows better compression, but consumes more
  160. * memory. \n The rough formula of memory used for temporary input
  161. * storage is `3 << lgBlock`.
  162. */
  163. BROTLI_PARAM_LGBLOCK = 3,
  164. /**
  165. * Flag that affects usage of "literal context modeling" format feature.
  166. *
  167. * This flag is a "decoding-speed vs compression ratio" trade-off.
  168. */
  169. BROTLI_PARAM_DISABLE_LITERAL_CONTEXT_MODELING = 4,
  170. /**
  171. * Estimated total input size for all ::BrotliEncoderCompressStream calls.
  172. *
  173. * The default value is 0, which means that the total input size is unknown.
  174. */
  175. BROTLI_PARAM_SIZE_HINT = 5,
  176. /**
  177. * Flag that determines if "Large Window Brotli" is used.
  178. */
  179. BROTLI_PARAM_LARGE_WINDOW = 6,
  180. /**
  181. * Recommended number of postfix bits (NPOSTFIX).
  182. *
  183. * Encoder may change this value.
  184. *
  185. * Range is from 0 to ::BROTLI_MAX_NPOSTFIX.
  186. */
  187. BROTLI_PARAM_NPOSTFIX = 7,
  188. /**
  189. * Recommended number of direct distance codes (NDIRECT).
  190. *
  191. * Encoder may change this value.
  192. *
  193. * Range is from 0 to (15 << NPOSTFIX) in steps of (1 << NPOSTFIX).
  194. */
  195. BROTLI_PARAM_NDIRECT = 8,
  196. /**
  197. * Number of bytes of input stream already processed by a different instance.
  198. *
  199. * @note It is important to configure all the encoder instances with same
  200. * parameters (except this one) in order to allow all the encoded parts
  201. * obey the same restrictions implied by header.
  202. *
  203. * If offset is not 0, then stream header is omitted.
  204. * In any case output start is byte aligned, so for proper streams stitching
  205. * "predecessor" stream must be flushed.
  206. *
  207. * Range is not artificially limited, but all the values greater or equal to
  208. * maximal window size have the same effect. Values greater than 2**30 are not
  209. * allowed.
  210. */
  211. BROTLI_PARAM_STREAM_OFFSET = 9
  212. } BrotliEncoderParameter;
  213. /**
  214. * Opaque structure that holds encoder state.
  215. *
  216. * Allocated and initialized with ::BrotliEncoderCreateInstance.
  217. * Cleaned up and deallocated with ::BrotliEncoderDestroyInstance.
  218. */
  219. typedef struct BrotliEncoderStateStruct BrotliEncoderState;
  220. /**
  221. * Sets the specified parameter to the given encoder instance.
  222. *
  223. * @param state encoder instance
  224. * @param param parameter to set
  225. * @param value new parameter value
  226. * @returns ::BROTLI_FALSE if parameter is unrecognized, or value is invalid
  227. * @returns ::BROTLI_FALSE if value of parameter can not be changed at current
  228. * encoder state (e.g. when encoding is started, window size might be
  229. * already encoded and therefore it is impossible to change it)
  230. * @returns ::BROTLI_TRUE if value is accepted
  231. * @warning invalid values might be accepted in case they would not break
  232. * encoding process.
  233. */
  234. BROTLI_ENC_API BROTLI_BOOL BrotliEncoderSetParameter(
  235. BrotliEncoderState* state, BrotliEncoderParameter param, uint32_t value);
  236. /**
  237. * Creates an instance of ::BrotliEncoderState and initializes it.
  238. *
  239. * @p alloc_func and @p free_func @b MUST be both zero or both non-zero. In the
  240. * case they are both zero, default memory allocators are used. @p opaque is
  241. * passed to @p alloc_func and @p free_func when they are called. @p free_func
  242. * has to return without doing anything when asked to free a NULL pointer.
  243. *
  244. * @param alloc_func custom memory allocation function
  245. * @param free_func custom memory free function
  246. * @param opaque custom memory manager handle
  247. * @returns @c 0 if instance can not be allocated or initialized
  248. * @returns pointer to initialized ::BrotliEncoderState otherwise
  249. */
  250. BROTLI_ENC_API BrotliEncoderState* BrotliEncoderCreateInstance(
  251. brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
  252. /**
  253. * Deinitializes and frees ::BrotliEncoderState instance.
  254. *
  255. * @param state decoder instance to be cleaned up and deallocated
  256. */
  257. BROTLI_ENC_API void BrotliEncoderDestroyInstance(BrotliEncoderState* state);
  258. /* Opaque type for pointer to different possible internal structures containing
  259. dictionary prepared for the encoder */
  260. typedef struct BrotliEncoderPreparedDictionaryStruct
  261. BrotliEncoderPreparedDictionary;
  262. /**
  263. * Prepares a shared dictionary from the given file format for the encoder.
  264. *
  265. * @p alloc_func and @p free_func @b MUST be both zero or both non-zero. In the
  266. * case they are both zero, default memory allocators are used. @p opaque is
  267. * passed to @p alloc_func and @p free_func when they are called. @p free_func
  268. * has to return without doing anything when asked to free a NULL pointer.
  269. *
  270. * @param type type of dictionary stored in data
  271. * @param data_size size of @p data buffer
  272. * @param data pointer to the dictionary data
  273. * @param quality the maximum Brotli quality to prepare the dictionary for,
  274. * use BROTLI_MAX_QUALITY by default
  275. * @param alloc_func custom memory allocation function
  276. * @param free_func custom memory free function
  277. * @param opaque custom memory manager handle
  278. */
  279. BROTLI_ENC_API BrotliEncoderPreparedDictionary*
  280. BrotliEncoderPrepareDictionary(BrotliSharedDictionaryType type,
  281. size_t data_size, const uint8_t data[BROTLI_ARRAY_PARAM(data_size)],
  282. int quality,
  283. brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque);
  284. BROTLI_ENC_API void BrotliEncoderDestroyPreparedDictionary(
  285. BrotliEncoderPreparedDictionary* dictionary);
  286. /**
  287. * Attaches a prepared dictionary of any type to the encoder. Can be used
  288. * multiple times to attach multiple dictionaries. The dictionary type was
  289. * determined by BrotliEncoderPrepareDictionary. Multiple raw prefix
  290. * dictionaries and/or max 1 serialized dictionary with custom words can be
  291. * attached.
  292. *
  293. * @returns ::BROTLI_FALSE in case of error
  294. * @returns ::BROTLI_TRUE otherwise
  295. */
  296. BROTLI_ENC_API BROTLI_BOOL BrotliEncoderAttachPreparedDictionary(
  297. BrotliEncoderState* state,
  298. const BrotliEncoderPreparedDictionary* dictionary);
  299. /**
  300. * Calculates the output size bound for the given @p input_size.
  301. *
  302. * @warning Result is only valid if quality is at least @c 2 and, in
  303. * case ::BrotliEncoderCompressStream was used, no flushes
  304. * (::BROTLI_OPERATION_FLUSH) were performed.
  305. *
  306. * @param input_size size of projected input
  307. * @returns @c 0 if result does not fit @c size_t
  308. */
  309. BROTLI_ENC_API size_t BrotliEncoderMaxCompressedSize(size_t input_size);
  310. /**
  311. * Performs one-shot memory-to-memory compression.
  312. *
  313. * Compresses the data in @p input_buffer into @p encoded_buffer, and sets
  314. * @p *encoded_size to the compressed length.
  315. *
  316. * @note If ::BrotliEncoderMaxCompressedSize(@p input_size) returns non-zero
  317. * value, then output is guaranteed to be no longer than that.
  318. *
  319. * @note If @p lgwin is greater than ::BROTLI_MAX_WINDOW_BITS then resulting
  320. * stream might be incompatible with RFC 7932; to decode such streams,
  321. * decoder should be configured with
  322. * ::BROTLI_DECODER_PARAM_LARGE_WINDOW = @c 1
  323. *
  324. * @param quality quality parameter value, e.g. ::BROTLI_DEFAULT_QUALITY
  325. * @param lgwin lgwin parameter value, e.g. ::BROTLI_DEFAULT_WINDOW
  326. * @param mode mode parameter value, e.g. ::BROTLI_DEFAULT_MODE
  327. * @param input_size size of @p input_buffer
  328. * @param input_buffer input data buffer with at least @p input_size
  329. * addressable bytes
  330. * @param[in, out] encoded_size @b in: size of @p encoded_buffer; \n
  331. * @b out: length of compressed data written to
  332. * @p encoded_buffer, or @c 0 if compression fails
  333. * @param encoded_buffer compressed data destination buffer
  334. * @returns ::BROTLI_FALSE in case of compression error
  335. * @returns ::BROTLI_FALSE if output buffer is too small
  336. * @returns ::BROTLI_TRUE otherwise
  337. */
  338. BROTLI_ENC_API BROTLI_BOOL BrotliEncoderCompress(
  339. int quality, int lgwin, BrotliEncoderMode mode, size_t input_size,
  340. const uint8_t input_buffer[BROTLI_ARRAY_PARAM(input_size)],
  341. size_t* encoded_size,
  342. uint8_t encoded_buffer[BROTLI_ARRAY_PARAM(*encoded_size)]);
  343. /**
  344. * Compresses input stream to output stream.
  345. *
  346. * The values @p *available_in and @p *available_out must specify the number of
  347. * bytes addressable at @p *next_in and @p *next_out respectively.
  348. * When @p *available_out is @c 0, @p next_out is allowed to be @c NULL.
  349. *
  350. * After each call, @p *available_in will be decremented by the amount of input
  351. * bytes consumed, and the @p *next_in pointer will be incremented by that
  352. * amount. Similarly, @p *available_out will be decremented by the amount of
  353. * output bytes written, and the @p *next_out pointer will be incremented by
  354. * that amount.
  355. *
  356. * @p total_out, if it is not a null-pointer, will be set to the number
  357. * of bytes compressed since the last @p state initialization.
  358. *
  359. *
  360. *
  361. * Internally workflow consists of 3 tasks:
  362. * -# (optionally) copy input data to internal buffer
  363. * -# actually compress data and (optionally) store it to internal buffer
  364. * -# (optionally) copy compressed bytes from internal buffer to output stream
  365. *
  366. * Whenever all 3 tasks can't move forward anymore, or error occurs, this
  367. * method returns the control flow to caller.
  368. *
  369. * @p op is used to perform flush, finish the stream, or inject metadata block.
  370. * See ::BrotliEncoderOperation for more information.
  371. *
  372. * Flushing the stream means forcing encoding of all input passed to encoder and
  373. * completing the current output block, so it could be fully decoded by stream
  374. * decoder. To perform flush set @p op to ::BROTLI_OPERATION_FLUSH.
  375. * Under some circumstances (e.g. lack of output stream capacity) this operation
  376. * would require several calls to ::BrotliEncoderCompressStream. The method must
  377. * be called again until both input stream is depleted and encoder has no more
  378. * output (see ::BrotliEncoderHasMoreOutput) after the method is called.
  379. *
  380. * Finishing the stream means encoding of all input passed to encoder and
  381. * adding specific "final" marks, so stream decoder could determine that stream
  382. * is complete. To perform finish set @p op to ::BROTLI_OPERATION_FINISH.
  383. * Under some circumstances (e.g. lack of output stream capacity) this operation
  384. * would require several calls to ::BrotliEncoderCompressStream. The method must
  385. * be called again until both input stream is depleted and encoder has no more
  386. * output (see ::BrotliEncoderHasMoreOutput) after the method is called.
  387. *
  388. * @warning When flushing and finishing, @p op should not change until operation
  389. * is complete; input stream should not be swapped, reduced or
  390. * extended as well.
  391. *
  392. * @param state encoder instance
  393. * @param op requested operation
  394. * @param[in, out] available_in @b in: amount of available input; \n
  395. * @b out: amount of unused input
  396. * @param[in, out] next_in pointer to the next input byte
  397. * @param[in, out] available_out @b in: length of output buffer; \n
  398. * @b out: remaining size of output buffer
  399. * @param[in, out] next_out compressed output buffer cursor;
  400. * can be @c NULL if @p available_out is @c 0
  401. * @param[out] total_out number of bytes produced so far; can be @c NULL
  402. * @returns ::BROTLI_FALSE if there was an error
  403. * @returns ::BROTLI_TRUE otherwise
  404. */
  405. BROTLI_ENC_API BROTLI_BOOL BrotliEncoderCompressStream(
  406. BrotliEncoderState* state, BrotliEncoderOperation op, size_t* available_in,
  407. const uint8_t** next_in, size_t* available_out, uint8_t** next_out,
  408. size_t* total_out);
  409. /**
  410. * Checks if encoder instance reached the final state.
  411. *
  412. * @param state encoder instance
  413. * @returns ::BROTLI_TRUE if encoder is in a state where it reached the end of
  414. * the input and produced all of the output
  415. * @returns ::BROTLI_FALSE otherwise
  416. */
  417. BROTLI_ENC_API BROTLI_BOOL BrotliEncoderIsFinished(BrotliEncoderState* state);
  418. /**
  419. * Checks if encoder has more output.
  420. *
  421. * @param state encoder instance
  422. * @returns ::BROTLI_TRUE, if encoder has some unconsumed output
  423. * @returns ::BROTLI_FALSE otherwise
  424. */
  425. BROTLI_ENC_API BROTLI_BOOL BrotliEncoderHasMoreOutput(
  426. BrotliEncoderState* state);
  427. /**
  428. * Acquires pointer to internal output buffer.
  429. *
  430. * This method is used to make language bindings easier and more efficient:
  431. * -# push data to ::BrotliEncoderCompressStream,
  432. * until ::BrotliEncoderHasMoreOutput returns BROTLI_TRUE
  433. * -# use ::BrotliEncoderTakeOutput to peek bytes and copy to language-specific
  434. * entity
  435. *
  436. * Also this could be useful if there is an output stream that is able to
  437. * consume all the provided data (e.g. when data is saved to file system).
  438. *
  439. * @attention After every call to ::BrotliEncoderTakeOutput @p *size bytes of
  440. * output are considered consumed for all consecutive calls to the
  441. * instance methods; returned pointer becomes invalidated as well.
  442. *
  443. * @note Encoder output is not guaranteed to be contiguous. This means that
  444. * after the size-unrestricted call to ::BrotliEncoderTakeOutput,
  445. * immediate next call to ::BrotliEncoderTakeOutput may return more data.
  446. *
  447. * @param state encoder instance
  448. * @param[in, out] size @b in: number of bytes caller is ready to take, @c 0 if
  449. * any amount could be handled; \n
  450. * @b out: amount of data pointed by returned pointer and
  451. * considered consumed; \n
  452. * out value is never greater than in value, unless it is @c 0
  453. * @returns pointer to output data
  454. */
  455. BROTLI_ENC_API const uint8_t* BrotliEncoderTakeOutput(
  456. BrotliEncoderState* state, size_t* size);
  457. /* Returns the estimated peak memory usage (in bytes) of the BrotliCompress()
  458. function, not counting the memory needed for the input and output. */
  459. BROTLI_ENC_EXTRA_API size_t BrotliEncoderEstimatePeakMemoryUsage(
  460. int quality, int lgwin, size_t input_size);
  461. /* Returns 0 if dictionary is not valid; otherwise returns allocation size. */
  462. BROTLI_ENC_EXTRA_API size_t BrotliEncoderGetPreparedDictionarySize(
  463. const BrotliEncoderPreparedDictionary* dictionary);
  464. /**
  465. * Gets an encoder library version.
  466. *
  467. * Look at BROTLI_MAKE_HEX_VERSION for more information.
  468. */
  469. BROTLI_ENC_API uint32_t BrotliEncoderVersion(void);
  470. #if defined(__cplusplus) || defined(c_plusplus)
  471. } /* extern "C" */
  472. #endif
  473. #endif /* BROTLI_ENC_ENCODE_H_ */