Skip to content

Commit a8600ea

Browse files
piscisaureuspiscisaureus
committed
Utf8 encoder
1 parent 080ffb8 commit a8600ea

7 files changed

Lines changed: 609 additions & 0 deletions

File tree

‎benchmark_utf8.js‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
2+
var test_utf8_new = process.test_utf8_new;
3+
var test_utf8_old_hint = process.test_utf8_old_hint;
4+
var test_utf8_old_nohint = process.test_utf8_old_nohint;
5+
6+
function benchmark_all() {
7+
var size = 8;
8+
var iterations = 10000000;
9+
while (true) {
10+
var log = Math.floor(log10(iterations)),
11+
factor = Math.pow(10, log),
12+
mantissa = Math.round(iterations / factor),
13+
round_iterations = mantissa * factor;
14+
benchmark_shapes(size, round_iterations);
15+
if (size <= 16) {
16+
iterations /= 2;
17+
} else if (size <= 64) {
18+
iterations /= 4
19+
} else {
20+
iterations /= 8;
21+
}
22+
size *= 8;
23+
if (iterations < 1) return;
24+
}
25+
}
26+
27+
function benchmark_shapes(size, iterations) {
28+
var shapes = ["left_tailed", "right_tailed", "tree", "flat"];
29+
for (var i = 0; i < shapes.length; i++) {
30+
for (var j = 0; j <= 2; j++) {
31+
benchmark(size, shapes[i], j, iterations);
32+
}
33+
}
34+
}
35+
36+
function benchmark(size, shape, unicode, iterations) {
37+
var string = shape_generators[shape](size, unicode),
38+
i, start, end;
39+
40+
process.stdout.write("size: " + string.length + ", shape: " + shape);
41+
process.stdout.write(", content: " + ["ansi", "single nonansi", "mixed"][unicode]);
42+
process.stdout.write(", iterations: " + iterations);
43+
44+
process.stdout.write("\nnew: ");
45+
start = (new Date()).getTime();
46+
for (i = iterations - 1; i >= 0; i--)
47+
test_utf8_new(string);
48+
end = (new Date()).getTime();
49+
process.stdout.write((end - start) / 1000 + " s");
50+
51+
process.stdout.write("\told_nohint: ");
52+
start = (new Date()).getTime();
53+
for (i = iterations - 1; i >= 0; i--)
54+
test_utf8_old_nohint(string);
55+
end = (new Date()).getTime();
56+
process.stdout.write((end - start) / 1000 + " s");
57+
58+
process.stdout.write("\told_hint: ");
59+
start = (new Date()).getTime();
60+
for (i = iterations - 1; i >= 0; i--)
61+
test_utf8_old_hint(string);
62+
end = (new Date()).getTime();
63+
process.stdout.write((end - start) / 1000 + " s");
64+
65+
process.stdout.write("\n\n");
66+
}
67+
68+
/*
69+
* Helpers
70+
*/
71+
72+
function log10(n) {
73+
return Math.log(n) / Math.log(10);
74+
}
75+
76+
/*
77+
* Shape generators
78+
*/
79+
80+
function part(unicode) {
81+
if (unicode) return "ü1234567";
82+
else return "12345678";
83+
}
84+
85+
var shape_generators = {
86+
left_tailed: function(size, unicode) {
87+
var s = part(unicode);
88+
while (s.length < size) {
89+
s = part(unicode > 1) + s;
90+
}
91+
return s;
92+
},
93+
94+
right_tailed: function(size, unicode) {
95+
var s = part(unicode);
96+
while (s.length < size) {
97+
s = s + part(unicode > 1);
98+
}
99+
return s;
100+
},
101+
102+
tree: function(size, unicode) {
103+
s = part(unicode > 1);
104+
while (s.length < size) {
105+
s = s + s;
106+
}
107+
var s = s + part(unicode);
108+
return s;
109+
},
110+
111+
flat: function(size, unicode) {
112+
return new Buffer(shape_generators.tree(size, unicode)).toString();
113+
}
114+
};
115+
116+
benchmark_all();

‎deps/v8/include/v8.h‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,8 @@ namespace internal {
112112

113113
class Arguments;
114114
class Object;
115+
class String;
116+
class ConsString;
115117
class Heap;
116118
class HeapObject;
117119
class Isolate;
@@ -1248,6 +1250,11 @@ class String : public Primitive {
12481250
*/
12491251
V8EXPORT bool CanMakeExternal();
12501252

1253+
/**
1254+
* Returns true if the string has only ascii (0-127) characters.
1255+
*/
1256+
V8EXPORT bool HasOnlyAsciiChars();
1257+
12511258
/** Creates an undetectable string from the supplied ASCII or UTF-8 data.*/
12521259
V8EXPORT static Local<String> NewUndetectable(const char* data,
12531260
int length = -1);
@@ -1324,6 +1331,62 @@ class String : public Primitive {
13241331
void operator=(const Value&);
13251332
};
13261333

1334+
/**
1335+
* Provides direct access to string memory. The user has to be aware that
1336+
* each buffer returned might contain either 8-bit or 16-bit characters. As
1337+
* long as the iterator exists no other interaction with the v8 heap is
1338+
* allowed, because the heap might be in inconsistent state.
1339+
*/
1340+
class V8EXPORT Memory {
1341+
static const int kCurrentIsSecondTag = 1;
1342+
static const int kParentStackSize = 1024;
1343+
1344+
public:
1345+
static enum StorageType {
1346+
kNone = 0,
1347+
kAscii = 1,
1348+
kTwoByte = 2
1349+
};
1350+
explicit Memory(Handle<v8::Value> obj);
1351+
~Memory() {
1352+
if (ptr_ != NULL) {
1353+
rewind();
1354+
}
1355+
}
1356+
const void* operator*() { return ptr_; }
1357+
int length() { return length_; }
1358+
StorageType storage_type() { return storage_type_; }
1359+
bool Next() {
1360+
if (ptr_ != NULL) {
1361+
next();
1362+
}
1363+
return ptr_ != NULL;
1364+
}
1365+
1366+
1367+
private:
1368+
void next();
1369+
void rewind();
1370+
inline void down();
1371+
inline void set_flat(v8::internal::String* flat);
1372+
inline void set_end();
1373+
inline void push_parent(bool second);
1374+
inline void pop_parent();
1375+
1376+
const void* ptr_;
1377+
int length_;
1378+
StorageType storage_type_;
1379+
v8::internal::ConsString* current_;
1380+
intptr_t parent_;
1381+
bool did_visit_second_;
1382+
int depth_;
1383+
intptr_t parents_[kParentStackSize];
1384+
1385+
// Disallow copying and assigning.
1386+
Memory(const Memory&);
1387+
void operator=(const Memory&);
1388+
};
1389+
13271390
private:
13281391
V8EXPORT void VerifyExternalStringResource(ExternalStringResource* val) const;
13291392
V8EXPORT static void CheckCast(v8::Value* obj);

0 commit comments

Comments
 (0)