blob: b96b95aa50a799b98c0fbb296c90fdd9e8417114 [file] [log] [blame]
/******************************************************************************
@File PVRTUnicode.cpp
@Title PVRTUnicode
@Version @Version
@Copyright Copyright (c) Imagination Technologies Limited.
@Platform All
@Description A small collection of functions used to decode Unicode formats to
individual code points.
******************************************************************************/
#include "PVRTUnicode.h"
#include <string.h>
/****************************************************************************
** Constants
****************************************************************************/
const PVRTuint32 c_u32ReplChar = 0xFFFD;
#define VALID_ASCII 0x80
#define TAIL_MASK 0x3F
#define BYTES_PER_TAIL 6
#define UTF16_SURG_H_MARK 0xD800
#define UTF16_SURG_H_END 0xDBFF
#define UTF16_SURG_L_MARK 0xDC00
#define UTF16_SURG_L_END 0xDFFF
#define UNICODE_NONCHAR_MARK 0xFDD0
#define UNICODE_NONCHAR_END 0xFDEF
#define UNICODE_RESERVED 0xFFFE
#define UNICODE_MAX 0x10FFFF
#define MAX_LEN 0x8FFF
/****************************************************************************
** A table which allows quick lookup to determine the number of bytes of a
** UTF8 code point.
****************************************************************************/
const PVRTuint8 c_u8UTF8Lengths[256] =
{
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
3,3,3,3,3,3,3,3,0,0,0,0,0,0,0,0,
};
/****************************************************************************
** A table which allows quick lookup to determine whether a UTF8 sequence
** is 'overlong'.
****************************************************************************/
const PVRTuint32 c_u32MinVals[4] =
{
0x00000000, // 0 tail bytes
0x00000080, // 1 tail bytes
0x00000800, // 2 tail bytes
0x00010000, // 3 tail bytes
};
/*!***************************************************************************
@Function CheckGenericUnicode
@Input c32 A UTF32 character/Unicode code point
@Returns Success or failure.
@Description Checks that the decoded code point is valid.
*****************************************************************************/
static bool CheckGenericUnicode(PVRTuint32 c32)
{
// Check that this value isn't a UTF16 surrogate mask.
if(c32 >= UTF16_SURG_H_MARK && c32 <= UTF16_SURG_L_END)
return false;
// Check non-char values
if(c32 >= UNICODE_NONCHAR_MARK && c32 <= UNICODE_NONCHAR_END)
return false;
// Check reserved values
if((c32 & UNICODE_RESERVED) == UNICODE_RESERVED)
return false;
// Check max value.
if(c32 > UNICODE_MAX)
return false;
return true;
}
/*!***************************************************************************
@Function PVRTUnicodeUTF8ToUTF32
@Input pUTF8 A UTF8 string, which is null terminated.
@Output aUTF32 An array of Unicode code points.
@Returns Success or failure.
@Description Decodes a UTF8-encoded string in to Unicode code points
(UTF32). If pUTF8 is not null terminated, the results are
undefined.
*****************************************************************************/
EPVRTError PVRTUnicodeUTF8ToUTF32(const PVRTuint8* const pUTF8, CPVRTArray<PVRTuint32>& aUTF32)
{
unsigned int uiTailLen, uiIndex;
unsigned int uiBytes = (unsigned int) strlen((const char*)pUTF8);
PVRTuint32 c32;
const PVRTuint8* pC = pUTF8;
while(*pC)
{
// Quick optimisation for ASCII characters
while(*pC && *pC < VALID_ASCII)
{
aUTF32.Append(*pC++);
}
// Done
if(!*pC)
break;
c32 = *pC++;
uiTailLen = c_u8UTF8Lengths[c32];
// Check for invalid tail length. Maximum 4 bytes for each UTF8 character.
// Also check to make sure the tail length is inside the provided buffer.
if(uiTailLen == 0 || (pC + uiTailLen > pUTF8 + uiBytes))
return PVR_OVERFLOW;
c32 &= (TAIL_MASK >> uiTailLen); // Get the data out of the first byte. This depends on the length of the tail.
// Get the data out of each tail byte
uiIndex = 0;
while(uiIndex < uiTailLen)
{
if((pC[uiIndex] & 0xC0) != 0x80)
return PVR_FAIL; // Invalid tail byte!
c32 = (c32 << BYTES_PER_TAIL) + (pC[uiIndex] & TAIL_MASK);
uiIndex++;
}
pC += uiIndex;
// Check overlong values.
if(c32 < c_u32MinVals[uiTailLen])
return PVR_FAIL;
if(!CheckGenericUnicode(c32))
return PVR_FAIL;
// OK
aUTF32.Append(c32);
}
return PVR_SUCCESS;
}
/*!***************************************************************************
@Function PVRTUnicodeUTF16ToUTF32
@Input pUTF16 A UTF16 string, which is null terminated.
@Output aUTF32 An array of Unicode code points.
@Returns Success or failure.
@Description Decodes a UTF16-encoded string in to Unicode code points
(UTF32). If pUTF16 is not null terminated, the results are
undefined.
*****************************************************************************/
EPVRTError PVRTUnicodeUTF16ToUTF32(const PVRTuint16* const pUTF16, CPVRTArray<PVRTuint32>& aUTF32)
{
const PVRTuint16* pC = pUTF16;
// Determine the number of shorts
while(*++pC && (pC - pUTF16) < MAX_LEN);
unsigned int uiBufferLen = (unsigned int) (pC - pUTF16);
if(uiBufferLen == MAX_LEN)
return PVR_OVERFLOW; // Probably not NULL terminated.
// Reset to start.
pC = pUTF16;
PVRTuint32 c32;
while(*pC)
{
// Straight copy. We'll check for surrogate pairs next...
c32 = *pC++;
// Check surrogate pair
if(c32 >= UTF16_SURG_H_MARK && c32 <= UTF16_SURG_H_END)
{
// Make sure the next 2 bytes are in range...
if(pC + 1 > pUTF16 + uiBufferLen || *pC == 0)
return PVR_OVERFLOW;
// Check that the next value is in the low surrogate range
if(*pC < UTF16_SURG_L_MARK || *pC > UTF16_SURG_L_END)
return PVR_FAIL;
// Decode
c32 = ((c32 - UTF16_SURG_H_MARK) << 10) + (*pC - UTF16_SURG_L_MARK) + 0x10000;
pC++;
}
if(!CheckGenericUnicode(c32))
return PVR_FAIL;
// OK
aUTF32.Append(c32);
}
return PVR_SUCCESS;
}
/*!***************************************************************************
@Function PVRTUnicodeUTF8Length
@Input pUTF8 A UTF8 string, which is null terminated.
@Returns The length of the string, in Unicode code points.
@Description Calculates the length of a UTF8 string. If pUTF8 is
not null terminated, the results are undefined.
*****************************************************************************/
unsigned int PVRTUnicodeUTF8Length(const PVRTuint8* const pUTF8)
{
const PVRTuint8* pC = pUTF8;
unsigned int charCount = 0;
unsigned int mask;
while(*pC)
{
// Quick optimisation for ASCII characters
const PVRTuint8* pStart = pC;
while(*pC && *pC < VALID_ASCII)
pC++;
charCount += (unsigned int) (pC - pStart);
// Done
if(!*pC)
break;
mask = *pC & 0xF0;
switch(mask)
{
case 0xF0: pC++;
case 0xE0: pC++;
case 0xC0: pC++;
break;
default:
_ASSERT(!"Invalid tail byte!");
return 0;
}
pC++;
charCount++;
}
return charCount;
}
/*!***************************************************************************
@Function PVRTUnicodeUTF16Length
@Input pUTF16 A UTF16 string, which is null terminated.
@Returns The length of the string, in Unicode code points.
@Description Calculates the length of a UTF16 string.
If pUTF16 is not null terminated, the results are
undefined.
*****************************************************************************/
unsigned int PVRTUnicodeUTF16Length(const PVRTuint16* const pUTF16)
{
const PVRTuint16* pC = pUTF16;
unsigned int charCount = 0;
while(*pC && (pC - pUTF16) < MAX_LEN)
{
if( pC[0] >= UTF16_SURG_H_MARK && pC[0] <= UTF16_SURG_H_END
&& pC[1] >= UTF16_SURG_L_MARK && pC[0] <= UTF16_SURG_L_END)
{
pC += 2;
}
else
{
pC += 1;
}
charCount++;
}
return charCount;
}
/*!***************************************************************************
@Function PVRTUnicodeValidUTF8
@Input pUTF8 A UTF8 string, which is null terminated.
@Returns true or false
@Description Checks whether the encoding of a UTF8 string is valid.
If pUTF8 is not null terminated, the results are undefined.
*****************************************************************************/
bool PVRTUnicodeValidUTF8(const PVRTuint8* const pUTF8)
{
unsigned int uiTailLen, uiIndex;
unsigned int uiBytes = (unsigned int) strlen((const char*)pUTF8);
const PVRTuint8* pC = pUTF8;
while(*pC)
{
// Quick optimisation for ASCII characters
while(*pC && *pC < VALID_ASCII) pC++;
// Done?
if(!*pC)
break;
PVRTuint32 c32 = *pC++;
uiTailLen = c_u8UTF8Lengths[c32];
// Check for invalid tail length. Maximum 4 bytes for each UTF8 character.
// Also check to make sure the tail length is inside the provided buffer.
if(uiTailLen == 0 || (pC + uiTailLen > pUTF8 + uiBytes))
return false;
// Get the data out of each tail byte
uiIndex = 0;
while(uiIndex < uiTailLen)
{
if((pC[uiIndex] & 0xC0) != 0x80)
return false; // Invalid tail byte!
c32 = (c32 << BYTES_PER_TAIL) + (pC[uiIndex] & TAIL_MASK);
uiIndex++;
}
pC += uiIndex;
// Check overlong values.
if(c32 < c_u32MinVals[uiTailLen])
return false;
if(!CheckGenericUnicode(c32))
return false;
}
return true;
}
/*****************************************************************************
End of file (PVRTUnicode.cpp)
*****************************************************************************/