[PATCH v3 13/21] backports: use BACKPORT_DIR prefix on kconfig sources

From: Luis R. Rodriguez
Date: Tue Nov 11 2014 - 03:18:33 EST

Next message: Luis R. Rodriguez: "[PATCH v3 11/21] backports: modify kconfig parser to use bpid and target_dir"
Previous message: Luis R. Rodriguez: "[PATCH v3 21/21] backports: add full kernel integration support"
In reply to: Luis R. Rodriguez: "[PATCH v3 21/21] backports: add full kernel integration support"
Next in thread: Luis R. Rodriguez: "[PATCH v3 11/21] backports: modify kconfig parser to use bpid and target_dir"
Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]

From: "Luis R. Rodriguez" <mcgrof@xxxxxxxx>

This will allow us to do less work for built-in integration support.
When packaging we only use BACKPORT_DIR for Kconfig sources we control,
to avoid a large change we avoid doing this for other Kconfig files.

For integration we'll always have to use BACKPORT_DIR on kconfig sources.

Suporting requires a bit of self evaluation of the variable within our
kconfig library, ideally we'd have support for groking all variables
defined within the Kconfig setup but that requires quite a lot more
work as it means also parsing Makefiles and inheriting these definitions
within config symbols when used. Since we only define one right now and
its used for built-in support we deal with it ourselves for now.

Please consider the complexity of adding others, it doesn't seem like
there would be much need for others though. If others wanted to use a
different BACKPORT_DIR path other than 'backports' that would require
tweaking here, but we'll start by assuming no one will want to do that.

$ time /home/mcgrof/backports/devel/ckmake --allyesconfig
1 3.0.101 [ OK ]
2 3.1.10 [ OK ]
3 3.2.62 [ OK ]
4 3.3.8 [ OK ]
5 3.4.104 [ OK ]
6 3.5.7 [ OK ]
7 3.6.11 [ OK ]
8 3.7.10 [ OK ]
9 3.8.13 [ OK ]
10 3.9.11 [ OK ]
11 3.10.58 [ OK ]
12 3.11.10 [ OK ]
13 3.12.31 [ OK ]
14 3.13.11 [ OK ]
15 3.14.22 [ OK ]
16 3.15.10 [ OK ]
17 3.16.6 [ OK ]
18 3.17.1 [ OK ]
19 3.18-rc1 [ OK ]

real 42m41.825s
user 1187m38.880s
sys 141m11.760s

Signed-off-by: Luis R. Rodriguez <mcgrof@xxxxxxxx>
---
backport/Kconfig | 40 +++++++++++++++---------------
gentree.py | 18 ++++++++++++--
lib/kconfig.py | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++---
3 files changed, 107 insertions(+), 25 deletions(-)

diff --git a/backport/Kconfig b/backport/Kconfig
index 415db89..b729d47 100644
--- a/backport/Kconfig
+++ b/backport/Kconfig
@@ -1,5 +1,8 @@
mainmenu "Linux Backports from $BACKPORTED_KERNEL_NAME $BACKPORTED_KERNEL_VERSION (with backports $BACKPORTS_VERSION)"

+config BACKPORT_DIR
+ string
+ option env="BACKPORT_DIR"
config BACKPORTS_VERSION
string
option env="BACKPORTS_VERSION"
@@ -10,33 +13,32 @@ config BACKPORTED_KERNEL_NAME
string
option env="BACKPORTED_KERNEL_NAME"

-# these will be generated
-source "Kconfig.kernel"
-source "Kconfig.versions"
+source "$BACKPORT_DIR/Kconfig.kernel"
+source "$BACKPORT_DIR/Kconfig.versions"

# Packaging hacks
-source "Kconfig.package.hacks"
+source "$BACKPORT_DIR/Kconfig.package.hacks"

# this has the configuration for the backport code
-source "compat/Kconfig"
+source "$BACKPORT_DIR/compat/Kconfig"

# these are copied from the kernel
-source "net/wireless/Kconfig"
-source "net/mac80211/Kconfig"
-source "net/bluetooth/Kconfig"
-source "drivers/net/wireless/Kconfig"
-source "drivers/net/ethernet/Kconfig"
-source "drivers/net/usb/Kconfig"
+source "$BACKPORT_DIR/net/wireless/Kconfig"
+source "$BACKPORT_DIR/net/mac80211/Kconfig"
+source "$BACKPORT_DIR/net/bluetooth/Kconfig"
+source "$BACKPORT_DIR/drivers/net/wireless/Kconfig"
+source "$BACKPORT_DIR/drivers/net/ethernet/Kconfig"
+source "$BACKPORT_DIR/drivers/net/usb/Kconfig"

-source "drivers/ssb/Kconfig"
-source "drivers/bcma/Kconfig"
+source "$BACKPORT_DIR/drivers/ssb/Kconfig"
+source "$BACKPORT_DIR/drivers/bcma/Kconfig"

-source "net/nfc/Kconfig"
+source "$BACKPORT_DIR/net/nfc/Kconfig"

-source "drivers/media/Kconfig"
+source "$BACKPORT_DIR/drivers/media/Kconfig"

-source "net/ieee802154/Kconfig"
-source "net/mac802154/Kconfig"
-source "drivers/net/ieee802154/Kconfig"
+source "$BACKPORT_DIR/net/ieee802154/Kconfig"
+source "$BACKPORT_DIR/net/mac802154/Kconfig"
+source "$BACKPORT_DIR/drivers/net/ieee802154/Kconfig"

-source "drivers/usb/class/Kconfig"
+source "$BACKPORT_DIR/drivers/usb/class/Kconfig"
diff --git a/gentree.py b/gentree.py
index 2c08bba..7632b90 100755
--- a/gentree.py
+++ b/gentree.py
@@ -32,7 +32,8 @@ class Bp_Identity(object):
"""
def __init__(self, integrate=False, kconfig_prefix='CPTCFG_',
project_prefix='', project_dir='',
- target_dir='', target_dir_name=''):
+ target_dir='', target_dir_name='',
+ kconfig_source_var=None):
self.integrate = integrate
self.kconfig_prefix = kconfig_prefix
self.kconfig_prefix_resafe = re.escape(kconfig_prefix)
@@ -43,6 +44,11 @@ class Bp_Identity(object):
self.project_dir = project_dir
self.target_dir = target_dir
self.target_dir_name = target_dir_name
+ self.kconfig_source_var = kconfig_source_var
+ if self.kconfig_source_var:
+ self.kconfig_source_var_resafe = re.escape(self.kconfig_source_var)
+ else:
+ self.kconfig_source_var_resafe = None

def read_copy_list(copyfile):
"""
@@ -681,6 +687,7 @@ def _main():
project_dir = args.outdir,
target_dir = os.path.join(args.outdir, 'backports/'),
target_dir_name = 'backports/',
+ kconfig_source_var = '$BACKPORT_DIR',
)
else:
bpid = Bp_Identity(integrate = integrate,
@@ -689,6 +696,7 @@ def _main():
project_dir = args.outdir,
target_dir = args.outdir,
target_dir_name = '',
+ kconfig_source_var = '$BACKPORT_DIR',
)

def logwrite(msg):
@@ -820,6 +828,7 @@ def process(kerneldir, copy_list_file, git_revision=None,

disable_list = add_automatic_backports(args)
if disable_list:
+ # No need to verify_sources() as compat's Kconfig has no 'source' call
bpcfg = kconfig.ConfigTree(os.path.join(bpid.target_dir, 'compat', 'Kconfig'), bpid)
bpcfg.disable_symbols(disable_list)
git_debug_snapshot(args, 'Add automatic backports')
@@ -828,10 +837,15 @@ def process(kerneldir, copy_list_file, git_revision=None,

# some post-processing is required
configtree = kconfig.ConfigTree(os.path.join(bpid.target_dir, 'Kconfig'), bpid)
+ ignore=['Kconfig.kernel', 'Kconfig.versions']
+
+ configtree.verify_sources(ignore=ignore)
+ git_debug_snapshot(args, "verify sources on top level backports Kconfig")
+
orig_symbols = configtree.symbols()

logwrite('Modify Kconfig tree ...')
- configtree.prune_sources(ignore=['Kconfig.kernel', 'Kconfig.versions'])
+ configtree.prune_sources(ignore=ignore)
git_debug_snapshot(args, "prune Kconfig tree")

if not bpid.integrate:
diff --git a/lib/kconfig.py b/lib/kconfig.py
index ac96613..ca3af63 100644
--- a/lib/kconfig.py
+++ b/lib/kconfig.py
@@ -12,10 +12,41 @@ sel_line = re.compile(r'^(?P<spc>\s+)select\s+(?P<sym>[^\s]*)\s*$')
backport_line = re.compile(r'^\s+#(?P<key>[ch]-file|module-name)\s*(?P<name>.*)')

class ConfigTree(object):
+ """
+ If you suppport kernel integration you should use bpid.kconfig_source_var
+ and set that to the the variable name you have used on your Kconfig file
+ for use as a directory prefix. When packaging you only need to set this on
+ the Kconfig sources part of backports. For integration this variable will
+ be used to prefix all Kconfig sources taken from the kernel. This variable
+ is used by Kconfig to expand to bpid.target_dir_name. All kernel Kconfig
+ source entries use a full path based on bpid.project_dir, when using
+ integration a prefix is needed on backported Kconfig entries to
+ help ensure that the base directory used for backported code is
+ bpid.target_dir and not bpid.project_dir. To help with we provide a
+ verification / modifier, verify_sources(), of the Kconfig entries, you
+ run this before working on the ConfigTree with any other helper.
+
+ Verification is only needed when integrating and if the top level Kconfig
+ entries that have a source entry.
+ """
def __init__(self, rootfile, bpid):
self.bpid = bpid
self.rootfile = os.path.basename(rootfile)
- self.src_line = re.compile(r'^\s*source\s+"(?P<src>[^\s"]*)"?\s*$')
+ if self.bpid.kconfig_source_var:
+ self.src_line = re.compile(r'^\s*source\s+"(?P<kconfig_var>' + self.bpid.kconfig_source_var_resafe + ')?/?(?P<src>[^\s"]*)"?\s*$')
+ self.kconfig_var_line = re.compile(r'.*(?P<kconfig_var>' + self.bpid.kconfig_source_var_resafe + ')+/+')
+ else:
+ self.src_line = re.compile(r'^\s*source\s+"(?P<src>[^\s"]*)"?\s*$')
+ self.kconfig_var_line = None
+ self.verified = False
+ self.need_verification = False
+ if self.bpid.integrate:
+ for l in open(os.path.join(self.bpid.target_dir, rootfile), 'r'):
+ m = self.src_line.match(l)
+ mrel = src_line_rel.match(l)
+ if m or mrel:
+ self.need_verification = True
+ break

def _check_relative_source(self, f, l):
#
@@ -28,12 +59,17 @@ class ConfigTree(object):
if m:
raise Exception('File: %s uses relative kconfig source entries (line: \'%s\'), use full path with quotes' %
(os.path.join(self.bpid.target_dir, f), l))
- def _walk(self, f):
+ def _walk(self, f, first_pass=False):
+ if self.bpid.integrate:
+ if not self.bpid.kconfig_source_var and self.need_verification:
+ raise Exception("You enabled integration but haven't set bpid.kconfig_source_var, this seems incorrect")
+ if not first_pass and not self.verified and self.need_verification:
+ raise Exception("You must run verify_sources() first, we don't do that for you as you may want to ignore some source files")
yield f
for l in open(os.path.join(self.bpid.target_dir, f), 'r'):
m = self.src_line.match(l)
if m and os.path.exists(os.path.join(self.bpid.target_dir, m.group('src'))):
- for i in self._walk(m.group('src')):
+ for i in self._walk(m.group('src'), first_pass=first_pass):
yield i
else:
self._check_relative_source(f, l)
@@ -49,12 +85,42 @@ class ConfigTree(object):
continue
src = m.group('src')
if src in ignore or os.path.exists(os.path.join(self.bpid.target_dir, src)):
+ out += l
+ else:
+ out += '#' + l
+ outf = open(os.path.join(self.bpid.target_dir, nf), 'w')
+ outf.write(out)
+ outf.close()
+
+ def verify_sources(self, ignore=[]):
+ if not self.need_verification:
+ return
+ for nf in self._walk(self.rootfile, first_pass=True):
+ out = ''
+ for l in open(os.path.join(self.bpid.target_dir, nf), 'r'):
+ m = self.src_line.match(l)
+ if not m:
+ self._check_relative_source(nf, l)
out += l
+ continue
+ src = m.group('src')
+ k = self.kconfig_var_line.match(l)
+ if src in ignore or os.path.exists(os.path.join(self.bpid.target_dir, src)):
+ if k:
+ out += l
+ else:
+ out += 'source "%s/%s"\n' % (self.bpid.kconfig_source_var, m.group('src'))
else:
- out += '#' + l
+ if k:
+ out += '# source "' + self.bpid.kconfig_source_var + '/' + src + '"\n'
+ else:
+ out += '#' + l
outf = open(os.path.join(self.bpid.target_dir, nf), 'w')
outf.write(out)
outf.close()
+ # Now the kconfig_var is always required from now on
+ self.src_line = re.compile(r'^\s*source\s+"(?P<kconfig_var>' + self.bpid.kconfig_source_var_resafe + ')+/+(?P<src>[^\s"]*)"?\s*$')
+ self.verified = True

def prune_sources(self, ignore=[]):
self._prune_sources(self.rootfile, ignore)
--
2.1.1

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

Next message: Luis R. Rodriguez: "[PATCH v3 11/21] backports: modify kconfig parser to use bpid and target_dir"
Previous message: Luis R. Rodriguez: "[PATCH v3 21/21] backports: add full kernel integration support"
In reply to: Luis R. Rodriguez: "[PATCH v3 21/21] backports: add full kernel integration support"
Next in thread: Luis R. Rodriguez: "[PATCH v3 11/21] backports: modify kconfig parser to use bpid and target_dir"
Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]