Skip to content

Commit 058d242

Browse files
wukan1986wukan1986
committed
试着加hdf5
1 parent c78b8e4 commit 058d242

12 files changed

Lines changed: 204 additions & 17 deletions

File tree

‎ArchiveData/ArchiveData.csproj‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,10 @@
3232
<WarningLevel>4</WarningLevel>
3333
</PropertyGroup>
3434
<ItemGroup>
35+
<Reference Include="HDF.PInvoke, Version=1.10.0.0, Culture=neutral, PublicKeyToken=18793c1fcffe58a3, processorArchitecture=MSIL">
36+
<HintPath>..\packages\HDF.PInvoke.1.10.0.4\lib\Net45\HDF.PInvoke.dll</HintPath>
37+
<Private>True</Private>
38+
</Reference>
3539
<Reference Include="NLog, Version=4.0.0.0, Culture=neutral, PublicKeyToken=5120e14c03d0593c, processorArchitecture=MSIL">
3640
<SpecificVersion>False</SpecificVersion>
3741
<HintPath>..\packages\NLog.4.2.3\lib\net45\NLog.dll</HintPath>
@@ -61,6 +65,13 @@
6165
</ProjectReference>
6266
</ItemGroup>
6367
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
68+
<PropertyGroup>
69+
<PostBuildEvent>
70+
if not exist "$(TargetDir)bin32" md "$(TargetDir)bin32"
71+
xcopy /s /y "$(SolutionDir)packages\HDF.PInvoke.1.10.0.4\lib\native\bin32\*.*" "$(TargetDir)bin32"
72+
if not exist "$(TargetDir)bin64" md "$(TargetDir)bin64"
73+
xcopy /s /y "$(SolutionDir)packages\HDF.PInvoke.1.10.0.4\lib\native\bin64\*.*" "$(TargetDir)bin64"</PostBuildEvent>
74+
</PropertyGroup>
6475
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
6576
Other similar extension points exist, see Microsoft.Common.targets.
6677
<Target Name="BeforeBuild">

‎ArchiveData/packages.config‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
<?xml version="1.0" encoding="utf-8"?>
22
<packages>
3+
<package id="HDF.PInvoke" version="1.10.0.4" targetFramework="net45" />
34
<package id="NLog" version="4.2.3" targetFramework="net45" />
45
</packages>

‎DataDownload/DataDownload.csproj‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,12 @@
3535
<StartupObject />
3636
</PropertyGroup>
3737
<ItemGroup>
38-
<Reference Include="Newtonsoft.Json, Version=8.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
39-
<SpecificVersion>False</SpecificVersion>
40-
<HintPath>..\packages\Newtonsoft.Json.8.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
38+
<Reference Include="HDF.PInvoke, Version=1.10.0.0, Culture=neutral, PublicKeyToken=18793c1fcffe58a3, processorArchitecture=MSIL">
39+
<HintPath>..\packages\HDF.PInvoke.1.10.0.4\lib\Net45\HDF.PInvoke.dll</HintPath>
40+
<Private>True</Private>
41+
</Reference>
42+
<Reference Include="Newtonsoft.Json">
43+
<HintPath>..\packages\Newtonsoft.Json.9.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
4144
</Reference>
4245
<Reference Include="System" />
4346
<Reference Include="System.Core" />
@@ -59,6 +62,13 @@
5962
<None Include="packages.config" />
6063
</ItemGroup>
6164
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
65+
<PropertyGroup>
66+
<PostBuildEvent>
67+
if not exist "$(TargetDir)bin32" md "$(TargetDir)bin32"
68+
xcopy /s /y "$(SolutionDir)packages\HDF.PInvoke.1.10.0.4\lib\native\bin32\*.*" "$(TargetDir)bin32"
69+
if not exist "$(TargetDir)bin64" md "$(TargetDir)bin64"
70+
xcopy /s /y "$(SolutionDir)packages\HDF.PInvoke.1.10.0.4\lib\native\bin64\*.*" "$(TargetDir)bin64"</PostBuildEvent>
71+
</PropertyGroup>
6272
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
6373
Other similar extension points exist, see Microsoft.Common.targets.
6474
<Target Name="BeforeBuild">

‎DataDownload/Program.cs‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@
66
using System.Text;
77
using System.Threading.Tasks;
88

9+
using HDF.PInvoke;
10+
using System.Runtime.InteropServices;
11+
912
namespace DataDownload
1013
{
1114
class Program
@@ -24,5 +27,125 @@ static void Main(string[] args)
2427
Console.WriteLine(l.Item1 + l.Item2);
2528
}
2629
}
30+
31+
[StructLayout(LayoutKind.Sequential)]
32+
struct ComType
33+
{
34+
[MarshalAs(UnmanagedType.ByValTStr, SizeConst = 16)]
35+
public string Name;
36+
public int x_pos;
37+
public int y_pos;
38+
public float Mass;
39+
public double Temperature;
40+
}
41+
42+
static void Main_Read(string[] args)
43+
{
44+
int DATA_ARRAY_LENGTH = 5;
45+
46+
//var h5 = H5F.open(@"E:\HDF5\HDF5DotNet-src\examples\CSharpExample\CSharpExample1\table.h5", H5F.ACC_RDONLY);
47+
48+
//var h5 = H5F.open(@"D:\test.h5", H5F.ACC_RDONLY);
49+
//var h5 = H5F.open(@"E:\HDF5\Hdf5DotnetTools-master\ToolTest\bin\Debug\table.h5", H5F.ACC_RDONLY);
50+
var h5 = H5F.open(@"E:\HDF5\test_gzip.h5", H5F.ACC_RDONLY);
51+
52+
var dataset = H5D.open(h5, "trans_detail/20160929");
53+
var spaceid = H5D.get_space(dataset);
54+
var npoints = H5S.get_simple_extent_npoints(spaceid);
55+
//var dims = H5S.get_simple_extent_dims(spaceid);
56+
int rank = H5S.get_simple_extent_ndims(spaceid);
57+
58+
// 是不是不能用自己的type
59+
var dtype = H5D.get_type(dataset);
60+
var dtcls = H5T.get_class(dtype);
61+
var size = H5T.get_size(dtype);
62+
var sz = Marshal.SizeOf(typeof(ComType));
63+
64+
var dtype_n = H5T.get_nmembers(dtype);
65+
66+
for(uint i=0;i<dtype_n;++i)
67+
{
68+
var x = H5T.get_member_name(dtype, i);
69+
var x4 = Marshal.PtrToStringAnsi(x);
70+
var y = H5T.get_member_type(dtype, i);
71+
var z = H5T.get_class(y);
72+
var x1 = H5T.get_member_offset(dtype, i);
73+
var x3 = H5T.get_size(y);
74+
75+
Console.WriteLine(x4);
76+
Console.WriteLine(z);
77+
Console.WriteLine(x1);
78+
//var x2 = Marshal.OffsetOf(typeof(ComType), x4).ToInt32();
79+
//Console.WriteLine(x2);
80+
Console.WriteLine(x3);
81+
}
82+
83+
int ss1 = Marshal.SizeOf(typeof(ComType));
84+
IntPtr p = Marshal.AllocHGlobal(ss1*11);
85+
H5D.read(dataset, dtype, H5S.ALL,H5S.ALL,H5P.DEFAULT, p);
86+
var s = Marshal.PtrToStructure(p, typeof(ComType));
87+
Console.WriteLine(s);
88+
var s2 = Marshal.PtrToStructure(p+ss1, typeof(ComType));
89+
Console.WriteLine(s2);
90+
var s3 = Marshal.PtrToStructure(p + ss1*4, typeof(ComType));
91+
Console.WriteLine(s3);
92+
var s4 = Marshal.PtrToStructure(p + ss1 * 5, typeof(ComType));
93+
Console.WriteLine(s4);
94+
var s6 = Marshal.PtrToStructure(p + ss1 * 10, typeof(ComType));
95+
Console.WriteLine(s6);
96+
}
97+
98+
static void Main2222(string[] args)
99+
{
100+
var h5 = H5F.create(@"D:\test.h5", H5F.ACC_TRUNC);
101+
102+
var typeId = H5T.create(H5T.class_t.COMPOUND, new IntPtr(40));
103+
104+
var strtype = H5T.copy(H5T.C_S1);
105+
H5T.set_size(strtype, new IntPtr(16));
106+
107+
H5T.insert(typeId, "Name", new IntPtr(0), strtype);
108+
H5T.insert(typeId, "x_pos", new IntPtr(16), H5T.NATIVE_INT32);
109+
H5T.insert(typeId, "y_pos", new IntPtr(20), H5T.NATIVE_INT32);
110+
H5T.insert(typeId, "Mass", new IntPtr(24), H5T.NATIVE_FLOAT);
111+
H5T.insert(typeId, "Temperature", new IntPtr(32), H5T.NATIVE_DOUBLE);
112+
113+
ulong[] dims = new ulong[] { 10000 };
114+
ulong[] chunk_size = new ulong[] { 1000};
115+
116+
var spaceid = H5S.create_simple(dims.Length, dims, null);
117+
118+
119+
var dcpl = H5P.create(H5P.DATASET_CREATE);
120+
121+
H5P.set_layout(dcpl, H5D.layout_t.COMPACT);
122+
H5P.set_deflate(dcpl, 6);
123+
124+
H5P.set_chunk(dcpl, chunk_size.Length, chunk_size);
125+
126+
127+
128+
129+
130+
var datasetid = H5D.create(h5, "Table1", typeId, spaceid, H5P.DEFAULT, dcpl);
131+
132+
ComType ct = new ComType()
133+
{
134+
Name = "aabb",
135+
x_pos = 2,
136+
y_pos = 1,
137+
Mass = 1.24F,
138+
Temperature = 45.7,
139+
};
140+
141+
IntPtr p = Marshal.AllocHGlobal(40 * (int)dims[0]);
142+
Marshal.StructureToPtr(ct, p, false);
143+
144+
145+
146+
H5D.write(datasetid, typeId, spaceid, H5S.ALL, H5P.DEFAULT, p);
147+
148+
H5F.close(h5);
149+
}
27150
}
28151
}

‎DataDownload/packages.config‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
<?xml version="1.0" encoding="utf-8"?>
22
<packages>
3-
<package id="Newtonsoft.Json" version="8.0.2" targetFramework="net45" />
3+
<package id="HDF.PInvoke" version="1.10.0.4" targetFramework="net45" />
4+
<package id="Newtonsoft.Json" version="9.0.1" targetFramework="net45" />
45
</packages>

‎DataReceiver/App.config‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
</startup>
2020
<appSettings>
2121
<clear />
22+
<add key="OutputFormat" value="hdf5"/>
2223
<add key="DataPath" value="test\Data" />
2324
<add key="ConfigPath" value="test\订阅全部个股期权" />
2425

‎DataReceiver/DRTickWriter.cs‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,19 @@
99

1010
namespace DataReceiver
1111
{
12+
/// <summary>
13+
///
14+
/// </summary>
1215
public class DRTickWriter:TickWriter
1316
{
14-
public DRTickWriter(string path):base(path)
17+
public string OutputFormat;
18+
public bool IsHdf5;
19+
20+
public DRTickWriter(string path, string outputFormat):base(path)
1521
{
22+
OutputFormat = outputFormat;
1623

24+
IsHdf5 = string.Compare(OutputFormat, "pd0", true) != 0;
1725
}
1826

1927
// 目前先不处理港股的tickSize变化的那种行情
@@ -92,14 +100,22 @@ PbTick CreateTick(ref DepthMarketDataNClass pDepthMarketData, PbTickCodec codec)
92100

93101
public bool Write(ref DepthMarketDataNClass pDepthMarketData)
94102
{
95-
QuantBox.Data.Serializer.V2.TickWriter.WriterDataItem item;
96-
if (Items.TryGetValue(pDepthMarketData.Symbol, out item))
103+
if(IsHdf5)
97104
{
98-
item.Tick = CreateTick(ref pDepthMarketData, item.Serializer.Codec);
99-
base.Write(item, item.Tick);
105+
// 输出
100106
return true;
101107
}
102-
return false;
108+
else
109+
{
110+
QuantBox.Data.Serializer.V2.TickWriter.WriterDataItem item;
111+
if (Items.TryGetValue(pDepthMarketData.Symbol, out item))
112+
{
113+
item.Tick = CreateTick(ref pDepthMarketData, item.Serializer.Codec);
114+
base.Write(item, item.Tick);
115+
return true;
116+
}
117+
return false;
118+
}
103119
}
104120
}
105121
}

‎DataReceiver/DataReceiver.csproj‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,12 @@
3535
<StartupObject />
3636
</PropertyGroup>
3737
<ItemGroup>
38-
<Reference Include="Newtonsoft.Json, Version=8.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
39-
<SpecificVersion>False</SpecificVersion>
40-
<HintPath>..\packages\Newtonsoft.Json.8.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
38+
<Reference Include="HDF.PInvoke, Version=1.10.0.0, Culture=neutral, PublicKeyToken=18793c1fcffe58a3, processorArchitecture=MSIL">
39+
<HintPath>..\packages\HDF.PInvoke.1.10.0.4\lib\Net45\HDF.PInvoke.dll</HintPath>
40+
<Private>True</Private>
41+
</Reference>
42+
<Reference Include="Newtonsoft.Json">
43+
<HintPath>..\packages\Newtonsoft.Json.9.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
4144
</Reference>
4245
<Reference Include="NLog, Version=4.0.0.0, Culture=neutral, PublicKeyToken=5120e14c03d0593c, processorArchitecture=MSIL">
4346
<SpecificVersion>False</SpecificVersion>
@@ -101,7 +104,11 @@
101104
</ItemGroup>
102105
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
103106
<PropertyGroup>
104-
<PostBuildEvent>rem xcopy "$(SolutionDir)lib\*.dll" "$(TargetDir)" /Y</PostBuildEvent>
107+
<PostBuildEvent>rem xcopy "$(SolutionDir)lib\*.dll" "$(TargetDir)" /Y
108+
if not exist "$(TargetDir)bin32" md "$(TargetDir)bin32"
109+
xcopy /s /y "$(SolutionDir)packages\HDF.PInvoke.1.10.0.4\lib\native\bin32\*.*" "$(TargetDir)bin32"
110+
if not exist "$(TargetDir)bin64" md "$(TargetDir)bin64"
111+
xcopy /s /y "$(SolutionDir)packages\HDF.PInvoke.1.10.0.4\lib\native\bin64\*.*" "$(TargetDir)bin64"</PostBuildEvent>
105112
</PropertyGroup>
106113
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
107114
Other similar extension points exist, see Microsoft.Common.targets.

‎DataReceiver/Program.cs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ namespace DataReceiver
2222
/// </summary>
2323
class Program
2424
{
25+
public const string KEY_OutputFormat = "OutputFormat";
2526
public const string KEY_DataPath = "DataPath";
2627
public const string KEY_ConfigPath = "ConfigPath";
2728

@@ -38,11 +39,14 @@ class Program
3839

3940
static void Main(string[] args)
4041
{
41-
DRTickWriter TickWriter = new DRTickWriter(ConfigurationManager.AppSettings[KEY_DataPath]);
42+
DRTickWriter TickWriter = new DRTickWriter(
43+
ConfigurationManager.AppSettings[KEY_DataPath],
44+
ConfigurationManager.AppSettings[KEY_OutputFormat]
45+
);
4246

4347
DataReceiver dataReceiver = new DataReceiver();
4448
dataReceiver.TickWriter = TickWriter;
45-
49+
4650
dataReceiver.ConfigPath = ConfigurationManager.AppSettings[KEY_ConfigPath];
4751
dataReceiver.DataPath = ConfigurationManager.AppSettings[KEY_DataPath];
4852
dataReceiver.ConnectionConfigListFileName = ConfigurationManager.AppSettings[KEY_ConnectionConfigListFileName];

‎DataReceiver/packages.config‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
<?xml version="1.0" encoding="utf-8"?>
22
<packages>
3+
<package id="HDF.PInvoke" version="1.10.0.4" targetFramework="net45" />
34
<package id="Microsoft.Tpl.Dataflow" version="4.5.24" targetFramework="net45" />
4-
<package id="Newtonsoft.Json" version="8.0.2" targetFramework="net45" />
5+
<package id="Newtonsoft.Json" version="9.0.1" targetFramework="net45" />
56
<package id="NLog" version="4.2.3" targetFramework="net45" />
67
</packages>

0 commit comments

Comments
 (0)