• file
• revisions
• annotate
• diff
• comparison
• raw

intl/icu/source/i18n/bocsu.cpp@6474c204b198 (annotated)

intl/icu/source/i18n/bocsu.cpp

Wed, 31 Dec 2014 06:09:35 +0100

author

Michael Schloh von Bennewitz <michael@schloh.com>

date

Wed, 31 Dec 2014 06:09:35 +0100

changeset 0

6474c204b198

permissions

-rw-r--r--

Cloned upstream origin tor-browser at tor-browser-31.3.0esr-4.5-1-build1
revision ID fc1c9ff7c1b2defdbc039f12214767608f46423f for hacking purpose.

michael@0	1	/*
michael@0	2	*******************************************************************************
michael@0	3	* Copyright (C) 2001-2011, International Business Machines
michael@0	4	* Corporation and others. All Rights Reserved.
michael@0	5	*******************************************************************************
michael@0	6	* file name: bocsu.cpp
michael@0	7	* encoding: US-ASCII
michael@0	8	* tab size: 8 (not used)
michael@0	9	* indentation:4
michael@0	10	*
michael@0	11	* Author: Markus W. Scherer
michael@0	12	*
michael@0	13	* Modification history:
michael@0	14	* 05/18/2001 weiv Made into separate module
michael@0	15	*/
michael@0	16
michael@0	17
michael@0	18	#include "unicode/utypes.h"
michael@0	19
michael@0	20	#if !UCONFIG_NO_COLLATION
michael@0	21
michael@0	22	#include "unicode/bytestream.h"
michael@0	23	#include "unicode/utf16.h"
michael@0	24	#include "bocsu.h"
michael@0	25
michael@0	26	/*
michael@0	27	* encode one difference value -0x10ffff..+0x10ffff in 1..3 bytes,
michael@0	28	* preserving lexical order
michael@0	29	*/
michael@0	30	U_CFUNC uint8_t *
michael@0	31	u_writeDiff(int32_t diff, uint8_t *p) {
michael@0	32	if(diff>=SLOPE_REACH_NEG_1) {
michael@0	33	if(diff<=SLOPE_REACH_POS_1) {
michael@0	34	*p++=(uint8_t)(SLOPE_MIDDLE+diff);
michael@0	35	} else if(diff<=SLOPE_REACH_POS_2) {
michael@0	36	*p++=(uint8_t)(SLOPE_START_POS_2+(diff/SLOPE_TAIL_COUNT));
michael@0	37	*p++=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT);
michael@0	38	} else if(diff<=SLOPE_REACH_POS_3) {
michael@0	39	p[2]=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT);
michael@0	40	diff/=SLOPE_TAIL_COUNT;
michael@0	41	p[1]=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT);
michael@0	42	*p=(uint8_t)(SLOPE_START_POS_3+(diff/SLOPE_TAIL_COUNT));
michael@0	43	p+=3;
michael@0	44	} else {
michael@0	45	p[3]=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT);
michael@0	46	diff/=SLOPE_TAIL_COUNT;
michael@0	47	p[2]=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT);
michael@0	48	diff/=SLOPE_TAIL_COUNT;
michael@0	49	p[1]=(uint8_t)(SLOPE_MIN+diff%SLOPE_TAIL_COUNT);
michael@0	50	*p=SLOPE_MAX;
michael@0	51	p+=4;
michael@0	52	}
michael@0	53	} else {
michael@0	54	int32_t m;
michael@0	55
michael@0	56	if(diff>=SLOPE_REACH_NEG_2) {
michael@0	57	NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m);
michael@0	58	*p++=(uint8_t)(SLOPE_START_NEG_2+diff);
michael@0	59	*p++=(uint8_t)(SLOPE_MIN+m);
michael@0	60	} else if(diff>=SLOPE_REACH_NEG_3) {
michael@0	61	NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m);
michael@0	62	p[2]=(uint8_t)(SLOPE_MIN+m);
michael@0	63	NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m);
michael@0	64	p[1]=(uint8_t)(SLOPE_MIN+m);
michael@0	65	*p=(uint8_t)(SLOPE_START_NEG_3+diff);
michael@0	66	p+=3;
michael@0	67	} else {
michael@0	68	NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m);
michael@0	69	p[3]=(uint8_t)(SLOPE_MIN+m);
michael@0	70	NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m);
michael@0	71	p[2]=(uint8_t)(SLOPE_MIN+m);
michael@0	72	NEGDIVMOD(diff, SLOPE_TAIL_COUNT, m);
michael@0	73	p[1]=(uint8_t)(SLOPE_MIN+m);
michael@0	74	*p=SLOPE_MIN;
michael@0	75	p+=4;
michael@0	76	}
michael@0	77	}
michael@0	78	return p;
michael@0	79	}
michael@0	80
michael@0	81	/*
michael@0	82	* Encode the code points of a string as
michael@0	83	* a sequence of byte-encoded differences (slope detection),
michael@0	84	* preserving lexical order.
michael@0	85	*
michael@0	86	* Optimize the difference-taking for runs of Unicode text within
michael@0	87	* small scripts:
michael@0	88	*
michael@0	89	* Most small scripts are allocated within aligned 128-blocks of Unicode
michael@0	90	* code points. Lexical order is preserved if "prev" is always moved
michael@0	91	* into the middle of such a block.
michael@0	92	*
michael@0	93	* Additionally, "prev" is moved from anywhere in the Unihan
michael@0	94	* area into the middle of that area.
michael@0	95	* Note that the identical-level run in a sort key is generated from
michael@0	96	* NFD text - there are never Hangul characters included.
michael@0	97	*/
michael@0	98	U_CFUNC void
michael@0	99	u_writeIdenticalLevelRun(const UChar *s, int32_t length, icu::ByteSink &sink) {
michael@0	100	char scratch[64];
michael@0	101	int32_t capacity;
michael@0	102
michael@0	103	UChar32 prev=0;
michael@0	104	int32_t i=0;
michael@0	105	while(i<length) {
michael@0	106	char *buffer=sink.GetAppendBuffer(1, length*2, scratch, (int32_t)sizeof(scratch), &capacity);
michael@0	107	uint8_t *p;
michael@0	108	// We must have capacity>=SLOPE_MAX_BYTES in case u_writeDiff() writes that much,
michael@0	109	// but we do not want to force the sink.GetAppendBuffer() to allocate
michael@0	110	// for a large min_capacity because we might actually only write one byte.
michael@0	111	if(capacity<16) {
michael@0	112	buffer=scratch;
michael@0	113	capacity=(int32_t)sizeof(scratch);
michael@0	114	}
michael@0	115	p=reinterpret_cast<uint8_t *>(buffer);
michael@0	116	uint8_t *lastSafe=p+capacity-SLOPE_MAX_BYTES;
michael@0	117	while(i<length && p<=lastSafe) {
michael@0	118	if(prev<0x4e00 || prev>=0xa000) {
michael@0	119	prev=(prev&~0x7f)-SLOPE_REACH_NEG_1;
michael@0	120	} else {
michael@0	121	/*
michael@0	122	* Unihan U+4e00..U+9fa5:
michael@0	123	* double-bytes down from the upper end
michael@0	124	*/
michael@0	125	prev=0x9fff-SLOPE_REACH_POS_2;
michael@0	126	}
michael@0	127
michael@0	128	UChar32 c;
michael@0	129	U16_NEXT(s, i, length, c);
michael@0	130	p=u_writeDiff(c-prev, p);
michael@0	131	prev=c;
michael@0	132	}
michael@0	133	sink.Append(buffer, (int32_t)(p-reinterpret_cast<uint8_t *>(buffer)));
michael@0	134	}
michael@0	135	}
michael@0	136
michael@0	137	U_CFUNC int32_t
michael@0	138	u_writeIdenticalLevelRunTwoChars(UChar32 first, UChar32 second, uint8_t *p) {
michael@0	139	uint8_t *p0 = p;
michael@0	140	if(first<0x4e00 || first>=0xa000) {
michael@0	141	first=(first&~0x7f)-SLOPE_REACH_NEG_1;
michael@0	142	} else {
michael@0	143	/*
michael@0	144	* Unihan U+4e00..U+9fa5:
michael@0	145	* double-bytes down from the upper end
michael@0	146	*/
michael@0	147	first=0x9fff-SLOPE_REACH_POS_2;
michael@0	148	}
michael@0	149
michael@0	150	p=u_writeDiff(second-first, p);
michael@0	151	return (int32_t)(p-p0);
michael@0	152	}
michael@0	153
michael@0	154	#endif /* #if !UCONFIG_NO_COLLATION */