Skip to content

Commit 2ce94ae

Browse files
teoteo
committed
[core] Allow variables to control the expansion of an iteratorRole
This also implements range-based iterator role syntax and moves iterator role template expansion from the unmarshal phase to the template processing phase.
1 parent 9312e05 commit 2ce94ae

3 files changed

Lines changed: 175 additions & 45 deletions

File tree

‎core/workflow/iteratorrange.go‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
/*
2+
* === This file is part of ALICE O² ===
3+
*
4+
* Copyright 2018-2020 CERN and copyright holders of ALICE O².
5+
* Author: Teo Mrnjavac <teo.mrnjavac@cern.ch>
6+
*
7+
* This program is free software: you can redistribute it and/or modify
8+
* it under the terms of the GNU General Public License as published by
9+
* the Free Software Foundation, either version 3 of the License, or
10+
* (at your option) any later version.
11+
*
12+
* This program is distributed in the hope that it will be useful,
13+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
14+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15+
* GNU General Public License for more details.
16+
*
17+
* You should have received a copy of the GNU General Public License
18+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
19+
*
20+
* In applying this license CERN does not waive the privileges and
21+
* immunities granted to it by virtue of its status as an
22+
* Intergovernmental Organization or submit itself to any jurisdiction.
23+
*/
24+
25+
package workflow
26+
27+
import (
28+
"encoding/json"
29+
"strconv"
30+
texttemplate "text/template"
31+
32+
"github.com/AliceO2Group/Control/core/workflow/template"
33+
)
34+
35+
type iteratorRange interface {
36+
GetRange(varStack map[string]string) (ran []string, err error)
37+
GetVar() string
38+
}
39+
40+
type iteratorRangeExpr struct {
41+
Range string `yaml:"range"`
42+
Var string `yaml:"var"`
43+
}
44+
45+
func (f *iteratorRangeExpr) GetRange(varStack map[string]string) (ran []string, err error) {
46+
rangeObj := make([]string, 0)
47+
48+
fields := template.Fields{
49+
template.WrapPointer(&f.Range),
50+
}
51+
err = fields.Execute("", varStack, template.MakeJsonConversionFuncMap(), make(map[string]texttemplate.Template))
52+
if err != nil {
53+
return
54+
}
55+
56+
err = json.Unmarshal([]byte(f.Range), &rangeObj)
57+
ran = rangeObj
58+
return
59+
}
60+
61+
func (f *iteratorRangeExpr) GetVar() string {
62+
return f.Var
63+
}
64+
65+
66+
type iteratorRangeFor struct {
67+
Begin string `yaml:"begin"`
68+
End string `yaml:"end"`
69+
Var string `yaml:"var"`
70+
}
71+
72+
func (f *iteratorRangeFor) GetRange(varStack map[string]string) (ran []string, err error) {
73+
ran = make([]string, 0)
74+
75+
fields := template.Fields{
76+
template.WrapPointer(&f.Begin),
77+
template.WrapPointer(&f.End),
78+
}
79+
err = fields.Execute("", varStack, make(map[string]interface{}), make(map[string]texttemplate.Template))
80+
if err != nil {
81+
return
82+
}
83+
84+
var begin, end int
85+
begin, err = strconv.Atoi(f.Begin)
86+
if err != nil {
87+
return
88+
}
89+
end, err = strconv.Atoi(f.End)
90+
if err != nil {
91+
return
92+
}
93+
94+
for j := begin; j <= end; j++ {
95+
ran = append(ran, strconv.Itoa(j))
96+
}
97+
return
98+
}
99+
100+
func (f *iteratorRangeFor) GetVar() string {
101+
return f.Var
102+
}

‎core/workflow/iteratorrole.go‎

Lines changed: 62 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
/*
22
* === This file is part of ALICE O² ===
33
*
4-
* Copyright 2018 CERN and copyright holders of ALICE O².
4+
* Copyright 2018-2020 CERN and copyright holders of ALICE O².
55
* Author: Teo Mrnjavac <teo.mrnjavac@cern.ch>
66
*
77
* This program is free software: you can redistribute it and/or modify
@@ -26,9 +26,9 @@ package workflow
2626

2727
import (
2828
"errors"
29+
2930
"github.com/AliceO2Group/Control/common/gera"
3031
"github.com/AliceO2Group/Control/core/repos"
31-
"strconv"
3232

3333
"github.com/AliceO2Group/Control/core/task"
3434
"github.com/AliceO2Group/Control/core/task/constraint"
@@ -37,7 +37,7 @@ import (
3737

3838
type iteratorRole struct {
3939
aggregator
40-
For iteratorInfo `yaml:"for,omitempty"`
40+
For iteratorRange `yaml:"for,omitempty"`
4141
template roleTemplate
4242
}
4343

@@ -55,14 +55,46 @@ func (i *iteratorRole) UnmarshalYAML(unmarshal func(interface{}) error) (err err
5555
if err != nil {
5656
return
5757
}
58-
auxFor := struct {
59-
For iteratorInfo `yaml:"for"`
58+
59+
type _iteratorRangeUnion struct {
60+
Begin *string `yaml:"begin"`
61+
End *string `yaml:"end"`
62+
Var *string `yaml:"var"`
63+
Range *string `yaml:"range"`
64+
}
65+
auxForUnion := struct {
66+
For _iteratorRangeUnion `yaml:"for"`
6067
}{}
61-
err = unmarshal(&auxFor)
68+
err = unmarshal(&auxForUnion)
6269
if err != nil {
6370
return
6471
}
6572

73+
var forBlock iteratorRange
74+
switch {
75+
case auxForUnion.For.Begin != nil && auxForUnion.For.End != nil && auxForUnion.For.Var != nil:
76+
auxFor := struct {
77+
For *iteratorRangeFor `yaml:"for"`
78+
}{}
79+
err = unmarshal(&auxFor)
80+
if err != nil {
81+
return
82+
}
83+
forBlock = auxFor.For
84+
case auxForUnion.For.Range != nil && auxForUnion.For.Var != nil:
85+
auxFor := struct {
86+
For *iteratorRangeExpr `yaml:"for"`
87+
}{}
88+
err = unmarshal(&auxFor)
89+
if err != nil {
90+
return
91+
}
92+
forBlock = auxFor.For
93+
default:
94+
err = errors.New("invalid range specifier in iterator")
95+
return
96+
}
97+
6698
var template roleTemplate
6799
switch {
68100
case auxUnion.Roles != nil && auxUnion.Task == nil:
@@ -79,47 +111,13 @@ func (i *iteratorRole) UnmarshalYAML(unmarshal func(interface{}) error) (err err
79111
}
80112

81113
role.template = template
82-
role.For = auxFor.For
114+
role.For = forBlock
83115

84116
// FIXME: if Name does not contain {{ }}, we must bail!
85-
86-
err = role.expandTemplate()
87-
if err != nil {
88-
return
89-
}
90117
*i = role
91118
return
92119
}
93120

94-
type iteratorInfo struct {
95-
Begin int `yaml:"begin"`
96-
End int `yaml:"end"`
97-
Var string `yaml:"var"`
98-
}
99-
100-
func (f *iteratorInfo) UnmarshalYAML(unmarshal func(interface{}) error) (err error) {
101-
aux := struct{
102-
Begin string `yaml:"begin"`
103-
End string `yaml:"end"`
104-
Var string `yaml:"var"`
105-
}{}
106-
err = unmarshal(&aux)
107-
if err != nil {
108-
return
109-
}
110-
111-
f.Begin, err = strconv.Atoi(aux.Begin)
112-
if err != nil {
113-
return
114-
}
115-
f.End, err = strconv.Atoi(aux.End)
116-
if err != nil {
117-
return
118-
}
119-
f.Var = aux.Var
120-
return
121-
}
122-
123121
func (i *iteratorRole) GlobFilter(g glob.Glob) (rs []Role) {
124122
rs = make([]Role, 0)
125123
for _, chr := range i.Roles {
@@ -136,6 +134,11 @@ func (i *iteratorRole) ProcessTemplates(workflowRepo *repos.Repo) (err error) {
136134
return errors.New("role tree error when processing templates")
137135
}
138136

137+
err = i.expandTemplate()
138+
if err != nil {
139+
return
140+
}
141+
139142
for _, role := range i.Roles {
140143
err = role.ProcessTemplates(workflowRepo)
141144
if err != nil {
@@ -146,14 +149,28 @@ func (i *iteratorRole) ProcessTemplates(workflowRepo *repos.Repo) (err error) {
146149
}
147150

148151
func (i *iteratorRole) expandTemplate() (err error) {
149-
values := make(map[string]string)
152+
varStack := make(map[string]string)
153+
if parent := i.GetParent(); parent != nil {
154+
varStack, _ = gera.FlattenStack(
155+
parent.GetDefaults(),
156+
parent.GetVars(),
157+
parent.GetUserVars(),
158+
)
159+
}
150160

151161
roles := make([]Role, 0)
152162

153-
for j := i.For.Begin; j <= i.For.End; j++ {
154-
values[i.For.Var] = strconv.Itoa(j)
163+
var ran []string
164+
ran, err = i.For.GetRange(varStack)
165+
if err != nil {
166+
return
167+
}
168+
169+
for _, localValue := range ran {
170+
locals := make(map[string]string)
171+
locals[i.For.GetVar()] = localValue
155172
var newRole Role
156-
newRole, err = i.template.generateRole(values)
173+
newRole, err = i.template.generateRole(locals)
157174
if err != nil {
158175
return
159176
}

‎core/workflow/load.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,15 @@
2525
package workflow
2626

2727
import (
28+
"fmt"
2829
"io/ioutil"
30+
"os"
31+
"time"
2932

3033
"github.com/AliceO2Group/Control/core/repos"
3134
"github.com/AliceO2Group/Control/core/task"
3235
"github.com/AliceO2Group/Control/core/the"
36+
"github.com/k0kubun/pp"
3337
"gopkg.in/yaml.v2"
3438
)
3539

@@ -63,6 +67,10 @@ func Load(workflowPath string, parent Updatable, taskManager *task.Manager, user
6367
workflow = root
6468
//fmt.Println("unprocessed workflow:")
6569
//_, _ = pp.Println(workflow)
70+
timestamp := fmt.Sprintf("%f", float64(time.Now().UnixNano())/1e9)
71+
f, err := os.Create(fmt.Sprintf("wf-unprocessed-%s.json", timestamp))
72+
_, _ = pp.Fprintln(f, workflow)
73+
defer f.Close()
6674

6775
err = workflow.ProcessTemplates(workflowRepo)
6876
if err != nil {
@@ -72,6 +80,9 @@ func Load(workflowPath string, parent Updatable, taskManager *task.Manager, user
7280
log.WithField("path", workflowPath).Debug("workflow loaded")
7381
//fmt.Println("processed workflow:")
7482
//_, _ = pp.Println(workflow)
83+
g, err := os.Create(fmt.Sprintf("wf-processed-%s.json", timestamp))
84+
_, _ = pp.Fprintln(g, workflow)
85+
defer g.Close()
7586

7687
// Update class list
7788
taskClassesRequired := workflow.GetTaskClasses()

0 commit comments

Comments
 (0)