Skip to content
This repository was archived by the owner on Jan 12, 2026. It is now read-only.

Commit 8635180

Browse files
Taishi Nojimaericzinnikas
Taishi Nojima
authored and
ericzinnikas
committed
Bump version (#4)
* Pin specific Celery/Kombu/Redis versions (google#305) * in TurbiniaTaskResult, input_evidence doesn't have to be a list (google#309) * input_evidence doesn't have to be a list * Fix exception * current pypi version of google-cloud-storage (1.13.0) requires google-cloud-core 0.28.1 (before the rename of google.cloud.iam (core) to google.api_core.iam (google#315) * Use a link to a "parent Evidence" instead of subclassing (google#296) * parent evidence * undo some simplifications for the sake of a simpler CL * Add some serialization * update docstring * Initialize attribute * set parent evidence if Evidence type is context dependant * don't pass parent_evidence at instantiation * undo linter stuff * comments * fix aim lib breaking tests * typo * Print version on start 3 (google#320) * Add files via upload * Delete turbiniactl.py * Delete turbiniactl.py * Add files via upload * Delete turbiniactl.py * Add files via upload * Update turbiniactl.py * Caps * Quick update to evidence docstrings (google#317) ... to disambiguate between _preprocess() and preprocess(). * Add Job filters (google#247) * Add job filters * fix docstrings. * update docstring * Get jobs filters working with new job manager * Refactor out FilterJobObjects into new method * Update YAPF * remove missed confict markers * Docstrings and renaming * Migrate job graph generator to use new manager (google#321) * Update Evidence local_path when it's saved (google#319) * Pin google-cloud-storage to 1.13.0 (google#326) Fixes google#325 Looks like google-cloud-storage was updated in: googleapis/google-cloud-python#6741 Which just got released as 1.13.1: https://pypi.org/project/google-cloud-storage/#history * Set image export to process all partitions (google#324) * Add --partitions all to image_export invocations * Fix typo * Explicitly set saved_paths to list (google#323) * Move version print after log level setup (google#322) * Move version print after log level setup * Remove extra whitespace * update the pystyle link (google#333) * Undefined name: Define 'unicode' in Python 3 (google#337) * Undefined name: Define 'unicode' in Python 3 __unicode()__ was removed in Python 3 because all __str__ are Unicode. [flake8](http://flake8.pycqa.org) testing of https://github.com/google/turbinia on Python 3.7.1 $ __flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics__ ``` ./tools/turbinia_job_graph.py:47:40: F821 undefined name 'unicode' parser.add_argument('filename', type=unicode, help='where to save the file') ^ 1 F821 undefined name 'unicode' 1 ``` * Placate PyLint * Added PSQ timeout to 1 week (google#336) * Error when worker version doesn't match server google#307 (google#327) * Added turbina_version to TurbinaTask * First approach * Changed to no rise error and return instead * Restored the run from run_wrapper * Changed format of strings * Changed words fixed line too long * bump version
1 parent 4c0d91e commit 8635180

14 files changed

Lines changed: 288 additions & 51 deletions

‎docs/contributing.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ that every developer knows to (largely) expect the same coding style.
6969
#### Style guide
7070

7171
We primarily follow the
72-
[Google Python Style Guide](https://google-styleguide.googlecode.com/svn/trunk/pyguide.html).
72+
[Google Python Style Guide](https://google.github.io/styleguide/pyguide.html).
7373
Various Turbinia specific additions/variations are:
7474

7575
* Using two spaces instead of four

‎tools/turbinia_job_graph.py‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,12 @@
1919
import argparse
2020
import graphviz
2121

22-
from turbinia.jobs import get_jobs as turbinia_jobs
22+
from turbinia.jobs import manager as jobs_manager
23+
24+
try:
25+
unicode
26+
except NameError:
27+
unicode = str # pylint: disable=redefined-builtin
2328

2429

2530
def create_graph():
@@ -29,15 +34,15 @@ def create_graph():
2934
Instance of graphviz.dot.Digraph
3035
"""
3136
dot = graphviz.Digraph(comment='Turbinia Evidence graph', format='png')
32-
for job in turbinia_jobs():
33-
dot.node(job.name)
37+
for _, job in jobs_manager.JobsManager.GetJobs():
38+
dot.node(job.NAME)
3439
for evidence in job.evidence_input:
3540
dot.node(evidence.__name__, shape='box')
36-
dot.edge(evidence.__name__, job.name)
41+
dot.edge(evidence.__name__, job.NAME)
3742

3843
for evidence in job.evidence_output:
3944
dot.node(evidence.__name__, shape='box')
40-
dot.edge(job.name, evidence.__name__)
45+
dot.edge(job.NAME, evidence.__name__)
4146
return dot
4247

4348

‎turbinia/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
# limitations under the License.
1515
"""Main Turbinia application."""
1616

17-
__version__ = '20181004+fb20190103'
17+
__version__ = '20181004+fb20190109'
1818

1919

2020
class TurbiniaException(Exception):

‎turbinia/client.py‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -307,6 +307,8 @@ def format_task_status(
307307
task.get('name'), task.get('user'), task.get('worker_name'),
308308
success, status))
309309
saved_paths = task.get('saved_paths', [])
310+
if saved_paths is None:
311+
saved_paths = []
310312
for path in saved_paths:
311313
results.append('\t{0:s}'.format(path))
312314
else:
@@ -423,11 +425,16 @@ class TurbiniaServer(object):
423425
task_manager (TaskManager): An object to manage turbinia tasks.
424426
"""
425427

426-
def __init__(self):
427-
"""Initialize Turbinia Server."""
428+
def __init__(self, jobs_blacklist=None, jobs_whitelist=None):
429+
"""Initializes Turbinia Server.
430+
431+
Args:
432+
jobs_blacklist (Optional[list[str]]): Jobs we will exclude from running
433+
jobs_whitelist (Optional[list[str]]): The only Jobs we will include to run
434+
"""
428435
config.LoadConfig()
429436
self.task_manager = task_manager.get_task_manager()
430-
self.task_manager.setup()
437+
self.task_manager.setup(jobs_blacklist, jobs_whitelist)
431438

432439
def start(self):
433440
"""Start Turbinia Server."""

‎turbinia/evidence.py‎

Lines changed: 49 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
if config.TASK_MANAGER.lower() == 'psq':
3232
from turbinia.processors import google_cloud
3333

34-
3534
def evidence_decode(evidence_dict):
3635
"""Decode JSON into appropriate Evidence object.
3736
@@ -64,6 +63,8 @@ def evidence_decode(evidence_dict):
6463
'No Evidence object of type {0:s} in evidence module'.format(type_))
6564

6665
evidence.__dict__ = evidence_dict
66+
if evidence_dict['parent_evidence']:
67+
evidence.parent_evidence = evidence_decode(evidence_dict['parent_evidence'])
6768
return evidence
6869

6970

@@ -78,6 +79,8 @@ class Evidence(object):
7879
processing this evidence.
7980
cloud_only: Set to True for evidence types that can only be processed in a
8081
cloud environment, e.g. GoogleCloudDisk.
82+
context_dependent: Whether this evidence is required to be built upon the
83+
context of a parent evidence.
8184
copyable: Whether this evidence can be copied. This will be set to True for
8285
object types that we want to copy to/from storage (e.g. PlasoFile, but
8386
not RawDisk).
@@ -91,7 +94,10 @@ class Evidence(object):
9194
that created it, if appropriate).
9295
local_path: A string of the local_path to the evidence.
9396
tags: dict of extra tags associated with this evidence.
94-
request_id: The id of the request this evidence came from, if any
97+
request_id: The id of the request this evidence came from, if any.
98+
parent_evidence: The Evidence object that was used to generate this one, and
99+
which pre/post process methods we need to re-execute to access data
100+
relevant to us.
95101
"""
96102

97103
def __init__(
@@ -100,12 +106,14 @@ def __init__(
100106
"""Initialization for Evidence."""
101107
self.copyable = False
102108
self.config = {}
109+
self.context_dependent = False
103110
self.cloud_only = False
104111
self.description = description
105112
self.source = source
106113
self.local_path = local_path
107114
self.tags = tags if tags else {}
108115
self.request_id = request_id
116+
self.parent_evidence = None
109117

110118
# List of jobs that have processed this evidence
111119
self.processed_by = []
@@ -122,7 +130,10 @@ def __repr__(self):
122130

123131
def serialize(self):
124132
"""Return JSON serializable object."""
125-
return self.__dict__
133+
serialized_evidence = self.__dict__
134+
if self.parent_evidence:
135+
serialized_evidence['parent_evidence'] = self.parent_evidence.serialize()
136+
return serialized_evidence
126137

127138
def to_json(self):
128139
"""Convert object to JSON.
@@ -142,7 +153,7 @@ def to_json(self):
142153

143154
return serialized
144155

145-
def preprocess(self):
156+
def _preprocess(self):
146157
"""Preprocess this evidence prior to task running.
147158
148159
This gets run in the context of the local task execution on the worker
@@ -151,7 +162,7 @@ def preprocess(self):
151162
"""
152163
pass
153164

154-
def postprocess(self):
165+
def _postprocess(self):
155166
"""Postprocess this evidence after the task runs.
156167
157168
This gets run in the context of the local task execution on the worker
@@ -160,6 +171,28 @@ def postprocess(self):
160171
"""
161172
pass
162173

174+
def preprocess(self):
175+
"""Runs the possible parent's evidence preprocessing code, then ours.
176+
177+
This is a wrapper function that will call the chain of pre-processors
178+
starting with the most distant ancestor. After all of the ancestors have
179+
been processed, then we run our pre-processor.
180+
"""
181+
if self.parent_evidence:
182+
self.parent_evidence.preprocess()
183+
self._preprocess()
184+
185+
def postprocess(self):
186+
"""Runs our postprocessing code, then our possible parent's evidence.
187+
188+
This is is a wrapper function that will run our post-processor, and will
189+
then recurse down the chain of parent Evidence and run those post-processors
190+
in order.
191+
"""
192+
self._postprocess()
193+
if self.parent_evidence:
194+
self.parent_evidence.postprocess()
195+
163196

164197
class Directory(Evidence):
165198
"""Filesystem directory evidence."""
@@ -185,6 +218,13 @@ def __init__(
185218
self.size = size
186219
super(RawDisk, self).__init__(*args, **kwargs)
187220

221+
def _preprocess(self):
222+
self.loopdevice_path = mount_local.PreprocessLosetup(self.local_path)
223+
224+
def _postprocess(self):
225+
mount_local.PostprocessDeleteLosetup(self.loopdevice_path)
226+
self.loopdevice_path = None
227+
188228

189229
class EncryptedDisk(RawDisk):
190230
"""Encrypted disk file evidence.
@@ -224,10 +264,10 @@ def __init__(self, project=None, zone=None, disk_name=None, *args, **kwargs):
224264
super(GoogleCloudDisk, self).__init__(*args, **kwargs)
225265
self.cloud_only = True
226266

227-
def preprocess(self):
267+
def _preprocess(self):
228268
self.local_path = google_cloud.PreprocessAttachDisk(self.disk_name)
229269

230-
def postprocess(self):
270+
def _postprocess(self):
231271
google_cloud.PostprocessDetachDisk(self.disk_name, self.local_path)
232272
self.local_path = None
233273

@@ -249,14 +289,14 @@ def __init__(self, embedded_path=None, *args, **kwargs):
249289
self.embedded_path = embedded_path
250290
super(GoogleCloudDiskRawEmbedded, self).__init__(*args, **kwargs)
251291

252-
def preprocess(self):
292+
def _preprocess(self):
253293
self.local_path = google_cloud.PreprocessAttachDisk(self.disk_name)
254294
self.loopdevice_path = mount_local.PreprocessLosetup(self.local_path)
255295
self.mount_path = mount_local.PreprocessMountDisk(
256296
self.loopdevice_path, self.mount_partition)
257297
self.local_path = os.path.join(self.mount_path, self.embedded_path)
258298

259-
def postprocess(self):
299+
def _postprocess(self):
260300
google_cloud.PostprocessDetachDisk(self.disk_name, self.local_path)
261301
mount_local.PostprocessUnmountPath(self.mount_path)
262302
mount_local.PostprocessDeleteLosetup(self.loopdevice_path)

‎turbinia/jobs/manager.py‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,65 @@
1616

1717
from __future__ import unicode_literals
1818

19+
from turbinia import TurbiniaException
20+
1921

2022
class JobsManager(object):
2123
"""The jobs manager."""
2224

2325
_job_classes = {}
2426

27+
@classmethod
28+
def FilterJobNames(cls, job_names, jobs_blacklist=None, jobs_whitelist=None):
29+
"""Filters a list of job names against white/black lists.
30+
31+
jobs_whitelist and jobs_blacklist must not be specified at the same time.
32+
33+
Args:
34+
job_names (list[str]): The names of the job_names to filter.
35+
jobs_blacklist (Optional[list[str]]): Job names to exclude.
36+
jobs_whitelist (Optional[list[str]]): Job names to include.
37+
38+
Returns:
39+
list[str]: Job names
40+
41+
Raises:
42+
TurbiniaException if both jobs_blacklist and jobs_whitelist are specified.
43+
"""
44+
jobs_blacklist = jobs_blacklist if jobs_blacklist else []
45+
jobs_blacklist = [job.lower() for job in jobs_blacklist]
46+
jobs_whitelist = jobs_whitelist if jobs_whitelist else []
47+
jobs_whitelist = [job.lower() for job in jobs_whitelist]
48+
49+
if jobs_whitelist and jobs_blacklist:
50+
raise TurbiniaException(
51+
'jobs_whitelist and jobs_blacklist cannot be specified at the same '
52+
'time.')
53+
elif jobs_blacklist:
54+
return [job for job in job_names if job.lower() not in jobs_blacklist]
55+
elif jobs_whitelist:
56+
return [job for job in job_names if job.lower() in jobs_whitelist]
57+
else:
58+
return job_names
59+
60+
@classmethod
61+
def FilterJobObjects(cls, jobs, jobs_blacklist=None, jobs_whitelist=None):
62+
"""Filters a list of job objects against white/black lists.
63+
64+
jobs_whitelist and jobs_blacklist must not be specified at the same time.
65+
66+
Args:
67+
jobs (list[TurbiniaJob]): The jobs to filter.
68+
jobs_blacklist (Optional[list[str]]): Job names to exclude.
69+
jobs_whitelist (Optional[list[str]]): Job names to include.
70+
71+
Returns:
72+
list[TurbiniaJob]: Job objects
73+
"""
74+
job_names = [job.name.lower() for job in jobs]
75+
job_names = cls.FilterJobNames(job_names, jobs_blacklist, jobs_whitelist)
76+
return [job for job in jobs if job.name.lower() in job_names]
77+
2578
@classmethod
2679
def DeregisterJob(cls, job_class):
2780
"""Deregisters a job class.

‎turbinia/jobs/manager_test.py‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
import unittest
88

9+
from turbinia import TurbiniaException
910
from turbinia.jobs import interface
1011
from turbinia.jobs import manager
1112

@@ -94,6 +95,55 @@ def testGetJobInstances(self):
9495
for job in jobs:
9596
self.assertIsInstance(job, interface.TurbiniaJob)
9697

98+
def testFilterJobNamesEmptyLists(self):
99+
"""Test FilterJobNames() with no filters."""
100+
job_names = ['testjob1', 'testjob2']
101+
return_job_names = manager.JobsManager.FilterJobNames(
102+
job_names, jobs_blacklist=[], jobs_whitelist=[])
103+
self.assertListEqual(job_names, return_job_names)
104+
105+
def testFilterJobNamesBlackList(self):
106+
"""Test FilterJobNames() with jobs_blacklist."""
107+
job_names = ['testjob1', 'testjob2']
108+
return_job_names = manager.JobsManager.FilterJobNames(
109+
job_names, jobs_blacklist=[job_names[0]], jobs_whitelist=[])
110+
self.assertListEqual(job_names[1:], return_job_names)
111+
112+
def testFilterJobObjectsBlackList(self):
113+
"""Test FilterJobObjects() with jobs_blacklist and objects."""
114+
jobs = [TestJob1(), TestJob2()]
115+
return_jobs = manager.JobsManager.FilterJobObjects(
116+
jobs, jobs_blacklist=[jobs[0].name], jobs_whitelist=[])
117+
self.assertListEqual(jobs[1:], return_jobs)
118+
119+
def testFilterJobNamesWhiteList(self):
120+
"""Test FilterJobNames() with jobs_whitelist."""
121+
job_names = ['testjob1', 'testjob2']
122+
return_job_names = manager.JobsManager.FilterJobNames(
123+
job_names, jobs_blacklist=[], jobs_whitelist=[job_names[0]])
124+
self.assertListEqual(job_names[:1], return_job_names)
125+
126+
def testFilterJobObjectsWhiteList(self):
127+
"""Test FilterJobObjects() with jobs_whitelist."""
128+
jobs = [TestJob1(), TestJob2()]
129+
return_jobs = manager.JobsManager.FilterJobObjects(
130+
jobs, jobs_blacklist=[], jobs_whitelist=[jobs[1].name])
131+
self.assertListEqual(jobs[1:], return_jobs)
132+
133+
def testFilterJobNamesException(self):
134+
"""Test FilterJobNames() with both jobs_blacklist and jobs_whitelist."""
135+
job_names = ['testjob1', 'testjob2']
136+
self.assertRaises(
137+
TurbiniaException, manager.JobsManager.FilterJobNames, job_names,
138+
jobs_blacklist=['a'], jobs_whitelist=['b'])
139+
140+
def testFilterJobNamesMixedCase(self):
141+
"""Test FilterJobNames() with mixed case inputs."""
142+
job_names = ['testjob1', 'testjob2']
143+
return_job_names = manager.JobsManager.FilterJobNames(
144+
job_names, jobs_blacklist=[], jobs_whitelist=['TESTJOB1'])
145+
self.assertListEqual(job_names[:1], return_job_names)
146+
97147

98148
if __name__ == '__main__':
99149
unittest.main()

‎turbinia/lib/utils.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ def extract_artifacts(artifact_names, disk_path, output_dir):
4040

4141
image_export_cmd = [
4242
'image_export.py', '--artifact_filters', artifacts, '--write', output_dir,
43-
disk_path
43+
'--partitions', 'all', disk_path
4444
]
4545

4646
# TODO: Consider break the exec helper to gather stdin/err.

0 commit comments

Comments
 (0)