diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000..6b0b1270ff --- /dev/null +++ b/LICENSE @@ -0,0 +1,203 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/METADATA b/METADATA new file mode 100644 index 0000000000..5a16d1de54 --- /dev/null +++ b/METADATA @@ -0,0 +1,17 @@ +name: "kernel/build" +description: + "Build scripts to build Android Kernels." + +third_party { + url { + type: HOMEPAGE + value: "https://source.android.com/setup/build/building-kernels" + } + url { + type: GIT + value: "https://android.googlesource.com/kernel/build" + } + version: "" + last_upgrade_date { year: 2019 month: 2 day: 1 } + license_type: NOTICE +} diff --git a/MODULE_LICENSE_APACHE2 b/MODULE_LICENSE_APACHE2 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/NOTICE b/NOTICE new file mode 120000 index 0000000000..7a694c9699 --- /dev/null +++ b/NOTICE @@ -0,0 +1 @@ +LICENSE \ No newline at end of file diff --git a/OWNERS b/OWNERS new file mode 100644 index 0000000000..414594e3f6 --- /dev/null +++ b/OWNERS @@ -0,0 +1,12 @@ +set noparent + +# GKI Dr. No Enforcement is active on this branch. Approval of one of the Dr. +# No reviewers is required following a regular CodeReview+2 vote of a code +# reviewer. +# +# See the GKI release documentation (go/gki-dr-no) for further details. +# +# The expanded list of reviewers can be found at: +# https://android.googlesource.com/kernel/common/+/android-mainline/OWNERS_DrNo + +include kernel/common:android-mainline:/OWNERS_DrNo diff --git a/README.md b/README.md deleted file mode 100644 index 4fc1fe9b19..0000000000 --- a/README.md +++ /dev/null @@ -1,73 +0,0 @@ -| Branch | Device | Android version | Base tag | Link | -| :-: | :-: | :-: | :-: | :-: | -| alioth-r-oss | Mi 10S, Redmi K40 | Android R | LA.UM.9.12.r1-08000-SMxx50.0 | [alioth-r-oss](https://github.com/MiCode/kernel_build/tree/alioth-r-oss) | -| apollo-q-oss | Redmi K30S Ultra | Android Q | LA.UM.8.12.r1-10600-sm8250.0 | [apollo-q-oss](https://github.com/MiCode/kernel_build/tree/apollo-q-oss) | -| cas-q-oss | Mi 10 Ultra, Mi 10 Pro, Mi 10, Redmi K30 Pro | Android Q | LA.UM.8.12.r1-10600-sm8250.0 | [cas-q-oss](https://github.com/MiCode/kernel_build/tree/cas-q-oss) | -| cmi-r-oss | Redmi K30S Ultra, Mi 10 Ultra, Mi 10 Pro, Redmi K30 Pro, Mi 10 | Android R | LA.UM.9.12.r1-08000-SMxx50.0 | [cmi-r-oss](https://github.com/MiCode/kernel_build/tree/cmi-r-oss) | -| dagu-s-oss | Xiaomi Pad 5 pro 12.4 | Android S | LA.UM.9.12.r1-13300-SMxx50.QSSI12.0-1 | [dagu-s-oss](https://github.com/MiCode/kernel_build/tree/dagu-s-oss) | -| elish-r-oss | Xiaomi Pad 5 Pro, Xiaomi Pad 5 Pro 5G | Android R | LA.UM.9.12.r1-08000-SMxx50.0 | [elish-r-oss](https://github.com/MiCode/kernel_build/tree/elish-r-oss) | -| flame-u-oss | Redmi 14R 5G, POCO M7 5G | Android U | LA.VENDOR.1.0.r1-24600-WAIPIO.QSSI14.0-1 | [flame-u-oss](https://github.com/MiCode/kernel_build/tree/flame-u-oss) | -| gauguin-q-oss | Redmi Note 9 Pro | Android Q | LA.UM.8.13.r1-09200-SAIPAN.0 | [gauguin-q-oss](https://github.com/MiCode/kernel_build/tree/gauguin-q-oss) | -| gauguin-r-oss | Redmi Note 9 Pro | Android R | LA.UM.9.12.r1-08000-SMxx50.0 | [gauguin-r-oss](https://github.com/MiCode/kernel_build/tree/gauguin-r-oss) | -| haydn-r-oss | Redmi K40 Pro | Android R | LA.UM.9.14.r1-11500-LAHAINA.0 | [haydn-r-oss](https://github.com/MiCode/kernel_build/tree/haydn-r-oss) | -| lisa-r-oss | Mi 11 LE | Android R | LA.UM.9.14.r1-16700-LAHAINA.0 | [lisa-r-oss](https://github.com/MiCode/kernel_build/tree/lisa-r-oss) | -| lmi-q-oss | Redmi K30 Pro| Android Q | LA.UM.8.12.r1-06000-sm8250.0 | [lmi-q-oss](https://github.com/MiCode/kernel_build/tree/lmi-q-oss) | -| lime-q-oss | Redmi Note 9 4G,Redmi POCO M3 | Android Q | LA.UM.8.15.r1-06600-KAMORTA.0 | [lime-q-oss](https://github.com/MiCode/kernel_build/tree/lime-q-oss) | -| mona-r-oss | Xiaomi Civi | Android R | LA.UM.9.14.r1-11500-LAHAINA.0 | [mona-r-oss](https://github.com/MiCode/kernel_build/tree/mona-r-oss) | -| munch-s-oss | Redmi K40S | Android S | LA.UM.9.12.r1-13300-SMxx50.QSSI12.0-1 | [munch-s-oss](https://github.com/MiCode/kernel_build/tree/munch-s-oss) | -| picasso-q-oss | Redmi K30 5G | Android Q | LA.UM.8.13.r1-03300-SAIPAN.0 | [picasso-q-oss](https://github.com/MiCode/kernel_build/tree/picasso-q-oss) | -| picasso-r-oss | Mi 10 Lite 5G, Redmi K30 5G, Mi 10 Lite Zoom | Android R | LA.UM.9.12.r1-08000-SMxx50.0 | [picasso-r-oss](https://github.com/MiCode/kernel_build/tree/picasso-r-oss) | -| pipa-t-oss | Xiaomi Pad 6 | Android T | LA.UM.9.12.r1-13300-SMxx50.QSSI12.0-1 | [pipa-t-oss](https://github.com/MiCode/kernel_build/tree/pipa-t-oss) | -| psyche-r-oss | Mi 12X | Android R | LA.UM.9.12.r1-08000-SMxx50.0 | [psyche-r-oss](https://github.com/MiCode/kernel_build/tree/psyche-r-oss) | -| redwood-s-oss | Redmi Note 12 Pro Speed, POCO X5 Pro 5G | Android S | LA.UM.9.14.r1-18200-LAHAINA.QSSI12.0 | [redwood-s-oss](https://github.com/MiCode/kernel_build/tree/redwood-s-oss) | -| star-r-oss | Mi 11 Lite 5G, Mi 11 Pro, Mi 11 Ultra, MIX FOLD | Android R | LA.UM.9.14.r1-11500-LAHAINA.0 | [star-r-oss](https://github.com/MiCode/kernel_build/tree/star-r-oss) | -| taoyao-s-oss | Xiaomi 12 Lite | Android S | LA.UM.9.14.r1-18300.05-LAHAINA.QSSI12.0-1 | [taoyao-s-oss](https://github.com/MiCode/kernel_build/tree/taoyao-s-oss) | -| umi-q-oss | Mi 10 Pro, Mi 10 | Android Q | LA.UM.8.12.r1-06000-sm8250.0 | [umi-q-oss](https://github.com/MiCode/kernel_build/tree/umi-q-oss) | -| vangogh-q-oss | Mi 10 Lite 5G, Mi 10 Lite Zoom | Android Q | LA.UM.8.13.r1-04700-SAIPAN.0 | [vangogh-q-oss](https://github.com/MiCode/kernel_build/tree/vangogh-q-oss) | -| venus-r-oss | Mi 11 | Android R | LA.UM.9.14.r1-10000-LAHAINA.0 | [venus-r-oss](https://github.com/MiCode/kernel_build/tree/venus-r-oss) | -| veux-r-oss | Redmi Note 11E Pro, Redmi Note 11 Pro 5G, Redmi Note 11 Pro+ 5G | Android R | LA.UM.9.16.r1-07900-MANNAR.0-1 | [veux-r-oss](https://github.com/MiCode/kernel_build/tree/veux-r-oss) | -| vili-r-oss | Xiaomi 11T Pro | Android R | LA.UM.9.14.r1-16700-LAHAINA.0 | [vili-r-oss](https://github.com/MiCode/kernel_build/tree/vili-r-oss) | -| odin-r-oss | MIX 4 | Android R | LA.UM.9.14.r1-16700-LAHAINA.0 | [odin-r-oss](https://github.com/MiCode/kernel_build/tree/odin-r-oss) | -| zijin-s-oss | Xiaomi Civi 1S | Android S | LA.UM.9.14.r1-18300.05-LAHAINA.QSSI12.0-1 | [zijin-s-oss](https://github.com/MiCode/kernel_build/tree/zijin-s-oss) | -| sheng-u-oss | Xiaomi Pad 6S Pro 12.4 | Android U | LA.VENDOR.13.2.0.r1-14800-KAILUA.0-1.36233.3 | [sheng-u-oss](https://github.com/MiCode/kernel_build/tree/sheng-u-oss) | -| peridot-u-oss | Redmi Turbo 3 | Android U | AU_LINUX_ANDROID_LA.VENDOR.14.3.0.R1.00.00.00.000.092 | [peridot-u-oss](https://github.com/MiCode/kernel_build/tree/peridot-u-oss) | -| chenfeng-u-oss | Civi 4 pro | Android U | AU_LINUX_KERNEL.PLATFORM.3.0.R1.00.00.00.017.065 | [chenfeng-u-oss](https://github.com/MiCode/kernel_build/tree/chenfeng-u-oss) | -| breeze-u-oss | Redmi Note 13R | Android U | LA.VENDOR.1.0.r1-24300-WAIPIO.QSSI14.0-1 | [breeze-u-oss](https://github.com/MiCode/kernel_build/tree/breeze-u-oss) | -| moon-u-oss | POCO M6/Redmi 13 | Android U | t-alps-release-s0.mp1.tc8sp-cs3-V1.31 | [moon-u-oss](https://github.com/MiCode/kernel_build/tree/moon-u-oss) | -| goku-u-oss | Xiaomi MIX Fold 4 | Android U | AU_LINUX_ANDROID_LA.VENDOR.14.3.0.R1.00.00.00.000.093 | [goku-u-oss](https://github.com/MiCode/kernel_build/tree/goku-u-oss-test) | -| bsp-degas-u-oss | Xiaomi 14T | Android U | alps-mp-u0.mp1.tc8sp3-V1 | [bsp-degas-u-oss](https://github.com/MiCode/kernel_build/tree/bsp-degas-u-oss) | -| gale-s-oss | Redmi 13C，POCO C65 | Android T | t-alps-release-s0.mp1.tc8sp-cs2-V1.31 |[gale-s-oss](https://github.com/MiCode/kernel_build/tree/gale-s-oss) | -| spark-s-oss | Redmi Pad SE 4G, Redmi Pad SE 8.7 4G | Android U | t-alps-release-s0.mp1.tc8sp-cs3-V1.43 |[spark-s-oss](https://github.com/MiCode/kernel_build/tree/spark-s-oss) | -| malachite-u-oss | Redmi Note 14 Pro | Android U | t-alps-release-u0.mp1.tc8sp3-V1 |[malachite-u-oss](https://github.com/MiCode/kernel_build/tree/malachite-u-oss) | -| beryl-u-oss | POCO M7 Pro 5G / Redmi Note 14 5G | Android U | t-alps-release-s0.mp1.tc8sp-cs3-V1.67 |[beryl-u-oss](https://github.com/MiCode/kernel_build/tree/beryl-u-oss) | -| muyu-v-oss | Xiaomi Pad 7 Pro | Android U | LA.VENDOR.14.3.0.r1-14500-r1.0.r1_00042.0 |[muyu-v-oss](https://github.com/MiCode/kernel_build/tree/muyu-v-oss) | -| emerald_r-u-oss | Redmi Note 14S | Android U | alps-mp-s0.mp1.tc8sp2-cs1-xm.V1.104_xiaomi.s0mp1.k6789v1.64_P50 |[emerald_r-u-oss](https://github.com/MiCode/kernel_build/tree/emerald_r-u-oss) | -| bsp-zorn-v-oss | REDMI K80 | Android V | qcom-LA.VENDOR.14.3.0.r1-14500-r1.0.r1_00042.0 |[bsp-zorn-v-oss](https://github.com/MiCode/kernel_build/tree/bsp-zorn-v-oss) | -| uke-v-oss | Xiaomi Pad 7 | Android V | LA.VENDOR.14.3.0.r1-14500-r1.0.r1_00042.0 |[uke-v-oss](https://github.com/MiCode/kernel_build/tree/uke-v-oss) | -| bsp-miro-v-oss | REDMI K80 PRO | Android V | qcom-LA.VENDOR.14.3.0.r1-14500-r1.0.r1_00042.0 |[bsp-miro-v-oss](https://github.com/MiCode/kernel_build/tree/bsp-miro-v-oss) | -| warm-u-oss | Redmi A4 5G / POCO C75 5G | Android U | snapdragon-mid-2024-spf-1-0_r1.0.r1_00006.0 |[warm-u-oss](https://github.com/MiCode/kernel_build/tree/warm-u-oss) | -| onyx-v-oss | REDMI Turbo 4 Pro | Android V | LA.VENDOR.15.4.0.r1-15000-r1.0.r1_00044.1 |[onyx-v-oss](https://github.com/MiCode/kernel_build/tree/onyx-v-oss) | -| bixi-v-oss | Xiaomi MIX Flip 2 | Android V | LA.VENDOR.15.4.0.r1-10100-r1.0.r1_00025.1 |[bixi-v-oss](https://github.com/MiCode/kernel_build/tree/bixi-v-oss) | -| creek-v-oss | POCO M7 / Redmi 15 | Android V | Divar.LA.3.0.1 MPSS.HA.1.2-00060-KD_ALL_PACK-1 |[creek-v-oss](https://github.com/MiCode/kernel_build/tree/creek-v-oss) | -| dew-v-oss | REDMI 15C / POCO C85 | Android V | t-alps-release-v0.mp1.rc-V4 |[dew-v-oss](https://github.com/MiCode/kernel_build/tree/dew-v-oss) | -| bsp-klimt-v-oss | Xiaomi 15T Pro | Android V | t-alps-release-v0.mp1.rc-V12.13 |[bsp-klimt-v-oss](https://github.com/MiCode/kernel_build/tree/bsp-klimt-v-oss) | -| kunzite-v-oss | REDMI Note 15 SE 5G / REDMI Note 15 | Android V | LA.VENDOR.15.4.1.r1-06600-r1.0.r1_00017.0 |[kunzite-v-oss](https://github.com/MiCode/kernel_build/tree/kunzite-v-oss) | -| lapis-v-oss | REDMI Note 15 Pro 5G | Android V | bsp-t-alps-release-u0.mp1.tc8sp3-V1 |[lapis-v-oss](https://github.com/MiCode/kernel_build/tree/lapis-v-oss) | -| spring-v-oss | Redmi 15 5G / Redmi Note 15R / POCO M7 Plus 5G | Android V | LA.VENDOR.14.3.1.r1-03700-MANNAR.QSSI15.0-1.107589.1.111280.1 |[spring-v-oss](https://github.com/MiCode/kernel_build/tree/spring-v-oss) | -| tornado-v-oss | Redmi 15C 5G / POCO C85 5G / REDMI 15C 5G / REDMI 15R 5G | Android V | alps-release-t0.mp1.tc8sp2-V1.281.1 |[tornado-v-oss](https://github.com/MiCode/kernel_build/tree/tornado-v-oss) | -| dash-w-oss | REDMI Turbo 5 Max | Android W | base-v-t-alps-release-v0.mp1.rc-V13.94 |[dash-w-oss](https://github.com/MiCode/kernel_build/tree/dash-w-oss) | -| rock-u-oss | POCO M5 / Redmi 11 Prime | Android U | alps-mp-s0.mp1.tc8sp2-cs1-xm.V1.0.13 |[rock-u-oss](https://github.com/MiCode/kernel_build/tree/rock-u-oss) | -| annibale-w-oss | REDMI K90 / POCO F8 Pro | Android W | LA.VENDOR.15.4.0.r1-15000-r1.0.r1_00044.1 |[annibale-w-oss](https://github.com/MiCode/kernel_build/tree/annibale-w-oss) | -| charoite-v-oss | REDMI Note 15 Pro | Android V | t-alps-release-s0.mp1.tc8sp2-cs1-xm-V1.143.1 |[charoite-v-oss](https://github.com/MiCode/kernel_build/tree/charoite-v-oss) | -| flourite-v-oss | POCO M8 Pro 5G / REDMI Note 15 Pro+ / REDMI Note 15 Pro+ 5G Pro | Android V | LA.VENDOR.14.3.0.r1-14500-r1.0.r1_00042.0 |[flourite-v-oss](https://github.com/MiCode/kernel_build/tree/flourite-v-oss) | -| bsp-klee-w-oss | REDMI Trubo 5 | Android W | t-alps-release-v0.mp1.rc-V2 |[klee-w-oss](https://github.com/MiCode/kernel_build/tree/bsp-klee-w-oss) | -| yupei-w-oss | Xiaomi Pad 8 | Android W | LA.VENDOR.15.4.0.r1-15000-r1.0.r1_00044.1 |[yupei-w-oss](https://github.com/MiCode/kernel_build/tree/yupei-w-oss) | -| piano-w-oss | Xiaomi Pad 8 Pro | Android W | LA.VENDOR.15.4.0.r1-15000-r1.0.r1_00044.1 |[piano-w-oss](https://github.com/MiCode/kernel_build/tree/piano-w-oss) | -| yili-w-oss | REDMI K Pad 2 | Android W | t-alps-release-b0.mp1.rc-V10.25 |[yili-w-oss](https://github.com/MiCode/kernel_build/tree/yili-w-oss) | -| agate-u-oss | Xiaomi 11T | Android U | alps-mp-s0.mp1.tc8sp-cs2-V1.14 |[agate-u-oss](https://github.com/MiCode/kernel_build/tree/agate-u-oss) | -| frost-r-oss | POCO C40 | Android R | JR510_R_V0.5.6.R3.B0005.7-0297 |[frost-r-oss](https://github.com/MiCode/kernel_build/tree/frost-r-oss) | -| light-u-oss | POCO M4 5G/Redmi 10 5G/Redmi 11 Prime 5G/Redmi Note 11E | Android U | t-alps-release-s0.mp1.tc8sp2-cs1-xm-V1.0.13 |[light-u-oss](https://github.com/MiCode/kernel_build/tree/light-u-oss) | -| spinel-v-oss | REDMI Note 15 | Android U | alps-mp-s0.mp1.tc8sp2-cs1-xm.V1.143.1 |[spinel-v-oss](https://github.com/MiCode/kernel_build/tree/spinel-v-oss) | -| sunstone-u-oss | Redmi Note 12 5G / Redmi Note 12 5G / Redmi Note 12R | Android U | Snapdragon_Mid_2021.SPF.2.0 r2.0.c3_00004.3.145544.1 |[sunstone-u-oss](https://github.com/MiCode/kernel_build/tree/sunstone-u-oss) | -| guitar-w-oss | REDMI Pad 2 SE | Android W | snapdragon-mid-2022-spf-2-0-2_00004.0 |[guitar-w-oss](https://github.com/MiCode/kernel_build/tree/guitar-w-oss) | -| erhu-w-oss | REDMI Pad 2 SE 4G | Android W | snapdragon-mid-2022-spf-2-0-2_00004.0 |[erhu-w-oss](https://github.com/MiCode/kernel_build/tree/erhu-w-oss) | diff --git a/_setup_env.sh b/_setup_env.sh new file mode 100644 index 0000000000..b34f1b9f2c --- /dev/null +++ b/_setup_env.sh @@ -0,0 +1,164 @@ +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an implementation detail of build.sh and friends. Do not source +# directly as it will spoil your shell and make build.sh unusable. You have +# been warned! If you have a good reason to source the result of this file into +# a shell, please let kernel-team@android.com know and we are happy to help +# with your use case. + +[ -n "$_SETUP_ENV_SH_INCLUDED" ] && return || _SETUP_ENV_SH_INCLUDED=1 + +# TODO: Use a $(gettop) style method. +export ROOT_DIR=$(readlink -f $PWD) + +export BUILD_CONFIG=${BUILD_CONFIG:-build.config} + +# Helper function to let build.config files add command to PRE_DEFCONFIG_CMDS, EXTRA_CMDS, etc. +# Usage: append_cmd PRE_DEFCONFIG_CMDS 'the_cmd' +function append_cmd() { + if [ ! -z "${!1}" ]; then + eval "$1=\"${!1} && \$2\"" + else + eval "$1=\"\$2\"" + fi +} + +export KERNEL_DIR +# for case that KERNEL_DIR is not specified in environment +if [ -z "${KERNEL_DIR}" ]; then + # for the case that KERNEL_DIR is not specified in the BUILD_CONFIG file + # use the directory of the build config file as KERNEL_DIR + # for the case that KERNEL_DIR is specified in the BUILD_CONFIG file, + # or via the config files sourced, the value of KERNEL_DIR + # set here would be overwritten, and the specified value would be used. + build_config_path=$(realpath ${ROOT_DIR}/${BUILD_CONFIG}) + build_config_dir=$(dirname ${build_config_path}) + build_config_dir=${build_config_dir##${ROOT_DIR}/} + KERNEL_DIR="${build_config_dir}" + echo "= Set default KERNEL_DIR: ${KERNEL_DIR}" +else + echo "= User environment KERNEL_DIR: ${KERNEL_DIR}" +fi + +set -a +. ${ROOT_DIR}/${BUILD_CONFIG} +for fragment in ${BUILD_CONFIG_FRAGMENTS}; do + . ${ROOT_DIR}/${fragment} +done +set +a + +echo "= The final value for KERNEL_DIR: ${KERNEL_DIR}" + +export COMMON_OUT_DIR=$(readlink -m ${OUT_DIR:-${ROOT_DIR}/out${OUT_DIR_SUFFIX}/${BRANCH}}) +export OUT_DIR=$(readlink -m ${COMMON_OUT_DIR}/${KERNEL_DIR}) +export DIST_DIR=$(readlink -m ${DIST_DIR:-${COMMON_OUT_DIR}/dist}) +export UNSTRIPPED_DIR=${DIST_DIR}/unstripped +export UNSTRIPPED_MODULES_ARCHIVE=unstripped_modules.tar.gz + +echo "========================================================" +echo "= build config: ${ROOT_DIR}/${BUILD_CONFIG}" +cat ${ROOT_DIR}/${BUILD_CONFIG} + +export TZ=UTC +export LC_ALL=C +export SOURCE_DATE_EPOCH=$(git -C ${ROOT_DIR}/${KERNEL_DIR} log -1 --pretty=%ct) +export KBUILD_BUILD_TIMESTAMP="$(date -d @${SOURCE_DATE_EPOCH})" +export KBUILD_BUILD_HOST=build-host +export KBUILD_BUILD_USER=build-user +export KBUILD_BUILD_VERSION=1 + +# List of prebuilt directories shell variables to incorporate into PATH +PREBUILTS_PATHS=( +LINUX_GCC_CROSS_COMPILE_PREBUILTS_BIN +LINUX_GCC_CROSS_COMPILE_ARM32_PREBUILTS_BIN +LINUX_GCC_CROSS_COMPILE_COMPAT_PREBUILTS_BIN +CLANG_PREBUILT_BIN +LZ4_PREBUILTS_BIN +DTC_PREBUILTS_BIN +LIBUFDT_PREBUILTS_BIN +BUILDTOOLS_PREBUILT_BIN +) + +if [ "${HERMETIC_TOOLCHAIN:-0}" -eq 1 ]; then + HOST_TOOLS=${OUT_DIR}/host_tools + rm -rf ${HOST_TOOLS} + mkdir -p ${HOST_TOOLS} + for tool in \ + bash \ + git \ + perl \ + rsync \ + sh \ + tar \ + ${ADDITIONAL_HOST_TOOLS} + do + ln -sf $(which $tool) ${HOST_TOOLS} + done + PATH=${HOST_TOOLS} + + # use relative paths for file name references in the binaries + # (e.g. debug info) + export KCPPFLAGS="-ffile-prefix-map=${ROOT_DIR}/${KERNEL_DIR}/= -ffile-prefix-map=${ROOT_DIR}/=" + + # set the common sysroot + sysroot_flags+="--sysroot=${ROOT_DIR}/build/build-tools/sysroot " + + # add openssl (via boringssl) and other prebuilts into the lookup path + cflags+="-I${ROOT_DIR}/prebuilts/kernel-build-tools/linux-x86/include " + + # add openssl and further prebuilt libraries into the lookup path + ldflags+="-Wl,-rpath,${ROOT_DIR}/prebuilts/kernel-build-tools/linux-x86/lib64 " + ldflags+="-L ${ROOT_DIR}/prebuilts/kernel-build-tools/linux-x86/lib64 " + + # Have host compiler use LLD and compiler-rt. + ldflags+="-fuse-ld=lld --rtlib=compiler-rt" + + export HOSTCFLAGS="$sysroot_flags $cflags" + export HOSTLDFLAGS="$sysroot_flags $ldflags" +fi + +for PREBUILT_BIN in "${PREBUILTS_PATHS[@]}"; do + PREBUILT_BIN=\${${PREBUILT_BIN}} + eval PREBUILT_BIN="${PREBUILT_BIN}" + if [ -n "${PREBUILT_BIN}" ]; then + # Mitigate dup paths + PATH=${PATH//"${ROOT_DIR}\/${PREBUILT_BIN}:"} + PATH=${ROOT_DIR}/${PREBUILT_BIN}:${PATH} + fi +done +export PATH +unset PYTHONPATH +unset PYTHONHOME +unset PYTHONSTARTUP + +echo +echo "PATH=${PATH}" +echo + +# verifies that defconfig matches the DEFCONFIG +function check_defconfig() { + (cd ${OUT_DIR} && \ + make "${TOOL_ARGS[@]}" O=${OUT_DIR} savedefconfig) + [ "$ARCH" = "x86_64" -o "$ARCH" = "i386" ] && local ARCH=x86 + echo Verifying that savedefconfig matches ${KERNEL_DIR}/arch/${ARCH}/configs/${DEFCONFIG} + RES=0 + diff -u ${KERNEL_DIR}/arch/${ARCH}/configs/${DEFCONFIG} ${OUT_DIR}/defconfig >&2 || + RES=$? + if [ ${RES} -ne 0 ]; then + echo ERROR: savedefconfig does not match ${KERNEL_DIR}/arch/${ARCH}/configs/${DEFCONFIG} >&2 + fi + return ${RES} +} +export -f check_defconfig diff --git a/abi/.gitignore b/abi/.gitignore new file mode 100644 index 0000000000..cbfa6e6716 --- /dev/null +++ b/abi/.gitignore @@ -0,0 +1,4 @@ +abigail-* +elfutils-* +*.pyc +__pycache__ diff --git a/abi/README.md b/abi/README.md new file mode 100644 index 0000000000..adfb532e59 --- /dev/null +++ b/abi/README.md @@ -0,0 +1,628 @@ +ABI Monitoring for Android Kernels +================================== + +Overview +-------- +In order to stabilize the in-kernel ABI of Android kernels, the ABI Monitoring +tooling has been created to collect and compare ABI representations from +existing kernel binaries (vmlinux + modules). The tools can be used to track +and mitigate changes to said ABI. This document describes the tooling, the +process of collecting and analyzing ABI representations and how such +representations can be used to ensure stability of the in-kernel ABI. Lastly, +this document gives some details about the process of contributing changes to +the Android kernels. + +This directory contains the specific tools for the ABI analysis. It should be +used as part of the build scripts that are provided by this repository (see +`../build_abi.sh`). + +Process Description +------------------- + +Analyzing the kernel's ABI is done in multiple steps. Most of the steps can be +automated: + + 1. Acquire the toolchain, build scripts and kernel sources through `repo` + 2. Provide any prerequisites (e.g. libabigail) + 3. Build the kernel and its ABI representation + 4. Analyze ABI differences between the build and a reference + 5. Update the ABI representation (if required) + 6. Working with symbol lists + + +The following instructions work for any kernel that can be built using a +supported toolchain (i.e. a prebuilt Clang toolchain). There exist [`repo` +manifests](https://android.googlesource.com/kernel/manifest/+refs) for all +Android common kernel branches, for some upstream branches (e.g. +upstream-linux-4.19.y) and several device specific kernels that ensure the +correct toolchain is used when building a kernel distribution. + + +Using the ABI Monitoring tooling +-------------------------------- + +### 1. Acquire the toolchain, build scripts and kernel sources through repo + +Toolchain, build scripts (i.e. these scripts) and kernel sources can be +acquired with `repo`. For detailed documentation, refer to the corresponding +documentation on +[source.android.com](https://source.android.com/setup/build/building-kernels). + +To illustrate the process, the following steps use `common-android-mainline`, +an Android kernel branch that is kept up-to-date with the upstream Linux +releases. In order to obtain this branch via `repo`, execute + +``` + $ repo init -u https://android.googlesource.com/kernel/manifest -b common-android-mainline + $ repo sync +``` + +### 2. Provide any prerequisites + +The ABI tooling makes use of [libabigail](https://sourceware.org/libabigail/), +a library and collection of tools to analyze binaries. A suitable set of +prebuilt binaries comes along with the kernel-build-tools and will +automatically be used when using `build_abi.sh`. + +For utilizing the lower level tooling (such as `dump_abi`), please ensure to +add the kernel-build-tools to the `PATH`. + +### 3. Build the kernel and its ABI representation + +At this point you are ready to build a kernel with the correct toolchain and to +extract an ABI representation from its binaries (vmlinux + modules). + +Similar to the usual Android kernel build process (using `build.sh`), this step +requires running `build_abi.sh`. + +``` + $ BUILD_CONFIG=common/build.config.gki.aarch64 build/build_abi.sh +``` + +**NOTE**: `build_abi.sh` makes use of `build.sh` and therefore accepts the +same environment variables to customize the build. It also *requires* the same +variables that would need to be passed to `build.sh`, such as `BUILD_CONFIG`. + +That builds the kernel and extracts the ABI representation into the `out` +directory. In this case `out/android-mainline/dist/abi.xml` would be a symbolic +link to `out/android-mainline/dist/abi-.xml`. `id` is computed from +executing `git describe` against the kernel source tree. + +### 4. Analyze ABI differences between the build and a reference representation + +`build_abi.sh` is capable of analyzing and reporting any ABI differences when +a reference is provided via the environment variable `ABI_DEFINITION`. +`ABI_DEFINITION` should point to a reference file relative to the kernel source +tree and can be specified on the command line or (more commonly) as a value in +*build.config*. E.g. + +``` + $ BUILD_CONFIG=common/build.config.gki.aarch64 \ + ABI_DEFINITION=abi_gki_aarch64.xml \ + build/build_abi.sh +``` + +Above, the `build.config.gki.aarch64` defines the reference file (as +*abi_gki_aarch64.xml*) and therefore the analysis has been completed. If an +abidiff was executed, then `build_abi.sh` will print the location of the report +and identify any ABI breakage. If breakages are detected, then `build_abi.sh` +will terminate and return a non-zero exit code. + +### 5. Update the ABI representation (if required) + +To update the ABI dump, `build_abi.sh` can be invoked with the `--update` flag. +It will update the corresponding abi.xml file that is defined via the +build.config. It might also be useful to invoke the script with `--print-report` +to print the differences the update fixes. The report is useful to include in +the commit message when updating the abi.xml. + +### 6. Working with symbol lists + +`build_abi.sh` can be parameterized to filter symbols during extraction and +comparison with KMI (Kernel Module Interface) symbol lists. These are simple +plain text files that list relevant ABI kernel symbols. E.g. a symbol list file +with the following content would limit ABI analysis to the ELF symbols with the +names `symbol1` and `symbol2`: + +``` + [abi_symbol_list] + symbol1 + symbol2 +``` + +**NOTE**: Please refer to the [libabigail +documentation](https://sourceware.org/libabigail/manual/kmidiff.html#environment) +for details about the KMI symbol list file format. + +Changes to other ELF symbols would not be considered any longer unless they are +indirectly affecting symbols that are part of the KMI. A symbol list file can be +specified -- similar to the abi baseline file via `ABI_DEFINITION=` -- in the +corresponding `build.config` configuration file with `KMI_SYMBOL_LIST=` as a file +relative to the kernel source directory (`$KERNEL_DIR`). In order to allow a +certain level of organization, additional symbol list files can be specified by +using `ADDITIONAL_KMI_SYMBOL_LISTS=` in the `build.config`. Similarly, it refers +to symbol lists in the `$KERNEL_DIR` and multiple files need to be separated by +whitespace. + +In order to **create an initial symbol list or to update an existing one**, the +`build_abi.sh` script must be used with the `--update-symbol-list` parameter. + +When run with an appropriate configuration, it will build the kernel and extract +the symbols that are exported from vmlinux and GKI modules _and_ are required by +any other module in the tree. + +Consider `vmlinux` exporting the following symbols (usually done via the +EXPORT_SYMBOL* macros): + +``` + func1 + func2 + func3 +``` + +Also, consider there are two vendor modules `modA.ko` and `modB.ko` which +require the following symbols (i.e. `undefined` entries in the symbol table): + +``` + modA.ko: func1 func2 + modB.ko: func2` +``` + +From an ABI stability point of view we need to keep `func1` and `func2` stable +as these are used by an external module. On the contrary, while `func3` is +exported it is not actively used (i.e. required) by any module. The symbol list +would therefore contain `func1` and `func2` only. + +In order to create or update an existing symbol list, `build_abi.sh` must be +run as follows: + +``` + $ BUILD_CONFIG=path/to/build.config.device build/build_abi.sh --update-symbol-list +``` + +In this example, `build.config.device` must include several configuration options: + - `vmlinux` must be in the `FILES` list; + - `KMI_SYMBOL_LIST` must be set and pointing at the KMI symbol list to update; + - `GKI_MODULES_LIST` should be set and pointing at the list of GKI modules. This + path is usually `android/gki_aarch64_modules`. + +**NOTE**: the `GKI_MODULES_LIST` option must be set in all vendor/OEM +`build.config` configurations downstream, but *not* in the upstream GKI +`build.config.gki.*`. `GKI_MODULES_LIST` is used in downstream builds to +differentiate vendor/OEM modules from GKI modules, which is not necessary +in upstream GKI builds where all modules are GKI modules. + +Working with the lower level ABI tooling +---------------------------------------- + +Most users will need to use `build_abi.sh`. In some cases, it might be +necessary to work with the lower level ABI tooling directly. There are +currently two commands -- `dump_abi` and `diff_abi` -- that are available to +collect and compare ABI files. These commands are used by `build_abi.sh`. See +the following sections for their usages. + +### Creating ABI dumps from kernel trees + +Provided a linux kernel tree with built vmlinux and kernel modules, the tool +`dump_abi` creates an ABI representation using the selected ABI tool. As of now +there is only one option: 'libabigail' (default). A sample invocation looks as +follows: + +``` + $ dump_abi --linux-tree path/to/out --out-file /path/to/abi.xml +``` + +The file `abi.xml` will contain a combined textual ABI representation that can +be observed from vmlinux and the kernel modules in the given directory. This +file might be used for manual inspection, further analysis or as a reference +file to enforce ABI stability. + +### Comparing ABI dumps + +ABI dumps created by `dump_abi` can be compared with `diff_abi`. Ensure to use +the same abi-tool for `dump_abi` and `diff_abi`. A sample invocation looks like: + +``` + $ diff_abi --baseline abi1.xml --new abi2.xml --report report.out +``` + +The report created is tool specific, but generally lists ABI changes detected +that affect the kernel's module interface. The files specified as `baseline` +and `new` are ABI representations collected with `dump_abi`. `diff_abi` +propagates the exit code of the underlying tool and therefore returns a +non-zero value in case the ABIs compared are incompatible. + +### Using KMI symbol lists + +To filter dumps created with `dump_abi` use the parameter `--kmi-symbol-list` +that takes a path to a KMI symbol list file: + +``` + $ dump_abi --linux-tree path/to/out --out-file /path/to/abi.xml --kmi-symbol-list /path/to/symbol_list +``` + +The same parameter can also be used to restrict the symbols that `diff_abi` +compares. + +### Comparing Kernel Binaries against the GKI reference KMI + +While working on the GKI Kernel compliance, it might be useful to regularly +compare a local Kernel build to a reference GKI KMI representation without +having to use `build_abi.sh`. The tool `gki_check` is a lightweight tool to +do exactly that. Given a local Linux Kernel build tree, a sample invocation to +compare the local binaries' representation to e.g. the 5.4 representation: + +``` + $ build/abi/gki_check --linux-tree path/to/out/ --kernel-version 5.4 +``` + +`gki_check` uses parameter names consistent with `dump_abi` and `diff_abi`. +Hence, `--kmi-symbol-list path/to/kmi_symbol_list` can be used to limit that +comparison to allowed symbols by passing a KMI symbol list. + +**NOTE:** When comparing the ABI representations between the GKI Kernel and the +locally built kernel, there might be cases that ABI changes are reported that +are purely caused by modifications to the kernel configuration (such as adding +modules with =m) without any other relevant code changes. As those are still +breakages, they need to be worked out in the Android Common Kernels. Please +contact kernel-team@android.com for advice. + +Dealing with ABI breakages +-------------------------- + +As an example, the following patch introduces a very obvious ABI breakage: + +``` + diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h + index 5ed8f6292a53..f2ecb34c7645 100644 + --- a/include/linux/mm_types.h + +++ b/include/linux/mm_types.h + @@ -339,6 +339,7 @@ struct core_state { + struct kioctx_table; + struct mm_struct { + struct { + + int dummy; + struct vm_area_struct *mmap; /* list of VMAs */ + struct rb_root mm_rb; + u64 vmacache_seqnum; /* per-thread vmacache */ +``` + +Running `build_abi.sh` again with this patch applied, the tooling will exit with +a non-zero error code and will report an ABI difference similar to this: + +``` + Leaf changes summary: 1 artifact changed + Changed leaf types summary: 1 leaf type changed + Removed/Changed/Added functions summary: 0 Removed, 0 Changed, 0 Added function + Removed/Changed/Added variables summary: 0 Removed, 0 Changed, 0 Added variable + + 'struct mm_struct at mm_types.h:372:1' changed: + type size changed from 6848 to 6912 (in bits) + there are data member changes: + [...] +``` + +### How to fix a broken ABI on Android Gerrit + +If you didn't intentionally break the kernel ABI, then you need to investigate +via the Android Gerrit test log to identify the issue(s) reported by the tool. Most +common causes of breakages are added or deleted functions, changed data +structures or changes to the ABI by adding config options that lead to any of +the aforementioned. Most likely you want to start with addressing the issues +found by the tool. + +You can reproduce the KernelABI test locally by running the following command +with the same arguments that you would have run `build/build.sh` with. + +Example command for the GKI kernels: + +``` + $ BUILD_CONFIG=common/build.config.gki.aarch64 build/build_abi.sh +``` + +### Updating the Kernel ABI + +If you need to update the kernel ABI, then you must update the corresponding +`abi.xml` file in the kernel source tree. This is most conveniently done by +using `build/build_abi.sh` like so: + +``` + $ build/build_abi.sh --update --print-report +``` + +with the same arguments that you would have run `build/build.sh` with. This +updates the correct `abi.xml` in the source tree and prints the detected +differences. It is recommended to include the printed report in the commit +message (at least partially). + + +Android Kernel Branches with predefined ABI +------------------------------------------- + +Some kernel branches might come with golden ABI representations for Android as +part of their source distribution. These ABI representations are supposed to be +accurate and should reflect the result of `build_abi.sh` as if you would execute +it on your own. As the ABI is heavily influenced by various kernel configuration +options, these .xml files usually belong to a certain configuration. E.g. the +`common-android-mainline` branch contains an `abi_gki_aarch64.xml` that +corresponds to the build result when using the `build.config.gki.aarch64`. In +particular, `build.config.gki.aarch64` also refers to this file as its +`ABI_DEFINITION`. + +Such predefined ABI representations are used as a baseline definition when +comparing with `diff_abi` (s.a.). E.g. to validate a kernel patch in regards to +any changes to the ABI, create the ABI representation with the patch applied and +use `diff_abi` to compare it to the expected ABI for that particular source tree +/ configuration. + +Enforcing the KMI using module versioning +----------------------------------------- + +The GKI kernels use [module versioning +](https://www.kernel.org/doc/html/latest/kbuild/modules.html?highlight=modules%20symvers#module-versioning) +(`CONFIG_MODVERSIONS`) as an measure to enforce KMI compliance at runtime. +Module versioning can cause CRC mismatch failures at module load time if the +expected KMI of a module does not match the vmlinux KMI. For example, here is +a typical failure occuring at module load time due to a CRC mismatch for the +symbol `module_layout()`: + +``` + init: Loading module /lib/modules/kernel/.../XXX.ko with args "" + XXX: disagrees about version of symbol module_layout + init: Failed to insmod '/lib/modules/kernel/.../XXX.ko' with args '' +``` + +### Why do we need module versioning? + +Module versioning is useful for many reasons: + +1. It catches changes in data structure visibility. If modules can change + opaque data structures, i.e. data structures that are not part of the KMI, + modules will break after future changes to the structure. +2. It adds a run time check to avoid accidentally loading a module that is not + KMI compatible with the kernel. This prevents hard-to-debug runtime issues/ + kernel crashes that will show up in the future. +3. `abidiff` has some current limitations in identifying ABI differences in + certain convoluted cases (they are being worked on) that `CONFIG_MODVERSIONS` + can catch. + +As an example for (1), consider the [fwnode +](https://android.googlesource.com/kernel/common/+/987d0b5bcf096a478aaf96faf5a288b4c95e9d37/include/linux/device.h#598) +field in [struct device +](https://android.googlesource.com/kernel/common/+/987d0b5bcf096a478aaf96faf5a288b4c95e9d37/include/linux/device.h#535). +That field MUST be opaque to modules so that they cannot make changes to fields +of `device.->fw_node` or make assumptions about its size. + +However, if a module includes `` (directly or indirectly), then +the `fwnode` field in the `struct device` is no longer opaque to it. The module +can then make changes to `device->fwnode->dev` or `device->fwnode->ops`. That +is problematic for several reasons: + +1. It can break assumptions the core kernel code is making about its internal + data structures. +2. If a future kernel update changes the `struct fwnode_handle` (the data type + of `fwnode`), then the module will no longer work with the new kernel. + Moreover, `abidiff` will not show any differences because the module is + breaking the KMI by directly manipulating internal data structures in ways that + cannot be captured by only inspecting the binary representation as of now. + +Having module versioning enabled prevents all of these issues. + +### How to check for CRC mismatch without booting the device? + +In the meantime, any full kernel build with `CONFIG_MODVERSIONS` enabled will +generate a `Module.symvers` file as part of the normal build process. The file +has one line for every symbol exported by the kernel (`vmlinux`) and the +modules. Each line consists of the CRC value, symbol name, symbol namespace, +vmlinux/module name exporting the symbol and export type (EXPORT\_SYMBOL vs +EXPORT\_SYMBOL\_GPL). + +You can compare the `Module.symvers` files between the GKI build and your build +to check for any CRC differences in the symbols exported by `vmlinux`. If there +is a CRC value difference in any symbol exported by `vmlinux` **AND** is used +by one of the modules you load in your device, the module will fail to load. + +If you do not have all the build artifacts, but just have the vmlinux file of +the GKI kernel and your kernel, you can compare the CRC value for a specific +symbol by running the following command on both the kernels and comparing the +output: + +``` + $ nm /vmlinux | grep __crc_ +``` + +For example, to check the CRC value for the `module_layout` symbol, + +``` + $ nm vmlinux | grep __crc_module_layout + 0000000008663742 A __crc_module_layout +``` + +### How to fix CRC mismatch? + +If you get a CRC mismatch when loading the module, here is how to you fix it: + +1. Build the GKI and your kernels, but add the `KBUILD_SYMTYPES=1` in front of + the command you use to build the kernel, if needed. Note that `build_abi.sh` + does this already. This will generate a `.symtypes` files for each `.o` file. + For example: + + ``` + $ KBUILD_SYMTYPES=1 \ + BUILD_CONFIG=common/build.config.gki.aarch64 build/build.sh + ``` + +2. Find the `.c` file in which the symbol with CRC mismatch is exported. For example: + + ``` + $ cd common && git grep EXPORT_SYMBOL.*module_layout + kernel/module.c:EXPORT_SYMBOL(module_layout); + ``` + +3. That `.c` file will have a corresponding `.symtypes` file in the GKI and + your kernel built artifacts. + + ``` + $ cd out/$BRANCH/common && ls -1 kernel/module.* + kernel/module.o + kernel/module.o.symversions + kernel/module.symtypes + ``` + + a. The format of this file is one (potentially very long) line per symbol. + + b. `[s|u|e|etc]#` at the start of the line means the symbol is of data type + [struct|union|enum|etc]. For example: + + ``` + t#bool typedef _Bool bool + ``` + + c. A missing '#' prefix in the start of the line indicates the symbol is + a function. For example: + + ``` + find_module s#module * find_module ( const char * ) + ``` + +4. Compare those two files and fix all the differences. + + **NOTE:** if you use vimdiff, `:set wrap` is recommended + +#### Case 1: Differences due to data type visibility + +If one kernel keeps a symbol/data type opaque to the modules and the +other kernel does not, then it shows up as a difference between the `.symtypes` +files of the two kernels. The `.symtypes` file from one of the kernels will +have `UNKNOWN` for a symbol and the other `.symtypes` file will have an +expanded view of the symbol/data type. + +Say you add this line to `include/linux/device.h` in your kernel: + +``` + #include +``` + +That will cause CRC mismatches and one of them would be for `module_layout()`. +If you compare the `module.symtypes` for that symbol, it will look like this: + +``` + $ diff -u /kernel/module.symtypes \ + /kernel/module.symtypes + --- /kernel/module.symtypes + +++ /kernel/module.symtypes + @@ -334,12 +334,15 @@ + ... + -s#fwnode_handle struct fwnode_handle { UNKNOWN } + +s#fwnode_reference_args struct fwnode_reference_args { s#fwnode_handle * fwnode ; unsigned int nargs ; t#u64 args [ 8 ] ; } + ... +``` + +If your kernel has it as `UNKNOWN` and the GKI kernel has the expanded view of +the symbol (very unlikely), then merge the latest Android Common Kernel into +your kernel so that you are using the latest GKI kernel base. + +In most instances, the GKI kernel has it as `UNKNOWN`, but your kernel has the +internal details of the symbol because of changes made to your kernel. This is +because one of the files in your kernel added a `#include` that is not present +in the GKI kernel. + +To identify the `#include` that causes the difference, follow these steps: + +1. Open the header file that defines the symbol/data type having this + difference. For example, `include/linux/fwnode.h` for the `struct + fwnode_handle`. +2. Add the following code at the top of the header file. + + ``` + #ifdef CRC_CATCH + #error "Included from here" + #endif + ``` + +3. Then in the module's `.c` file that has a CRC mismatch, add the following as + the first line before any of the #include lines. + + ``` + #define CRC_CATCH 1 + ``` + +4. Now compile your module. You will get a build time error that shows the chain + of header file `#include` that led to this CRC mismatch. + + ``` + In file included from .../drivers/clk/XXX.c:16: + In file included from .../include/linux/of_device.h:5: + In file included from .../include/linux/cpu.h:17: + In file included from .../include/linux/node.h:18: + .../include/linux/device.h:16:2: error: "Included from here" + #error "Included from here" + ``` + +5. One of the links in this chain of `#include` is due to a change done in your + kernel, that is missing in the GKI kernel. +6. Once you have identified the change, revert it in your kernel or [upload it to + ACK and get it merged](https://android.googlesource.com/kernel/common/+/987d0b5bcf096a478aaf96faf5a288b4c95e9d37/README.md). + +#### Case 2: Differences due to data type changes + +If the CRC mismatch for a symbol/data type is not due to a difference in +visibility, then it is due to actual changes (additions/removals/changes) in +the data type itself. Typically `abidiff` would have caught this, but if it +misses any due to known detection gaps, `CONFIG_MODVERSIONS` would catch it. + +Say you make this change in your kernel: + +``` + diff --git a/include/linux/iommu.h b/include/linux/iommu.h + --- a/include/linux/iommu.h + +++ b/include/linux/iommu.h + @@ -259,7 +259,7 @@ struct iommu_ops { + void (*iotlb_sync)(struct iommu_domain *domain); + phys_addr_t (*iova_to_phys)(struct iommu_domain *domain, dma_addr_t iova); + phys_addr_t (*iova_to_phys_hard)(struct iommu_domain *domain, + - dma_addr_t iova); + + dma_addr_t iova, unsigned long trans_flag); + int (*add_device)(struct device *dev); + void (*remove_device)(struct device *dev); + struct iommu_group *(*device_group)(struct device *dev); +``` + +That will cause a lot of CRC mismatches, but one of them would be for +`devm_of_platform_populate()`. + +If you compare the .symtypes for that symbol, it will look like this: + +``` + $ diff -u /drivers/of/platform.symtypes \ + /drivers/of/platform.symtypes + --- /drivers/of/platform.symtypes + +++ /drivers/of/platform.symtypes + @@ -399,7 +399,7 @@ + ... + -s#iommu_ops struct iommu_ops { ... ; t#phy + s_addr_t ( * iova_to_phys_hard ) ( s#iommu_domain * , t#dma_addr_t ) ; int + ( * add_device ) ( s#device * ) ; ... + +s#iommu_ops struct iommu_ops { ... ; t#phy + s_addr_t ( * iova_to_phys_hard ) ( s#iommu_domain * , t#dma_addr_t , unsigned long ) ; int ( * add_device ) ( s#device * ) ; ... +``` + +To identify the changed type, follow these steps: + +1. Find the definition of the symbol in the source code (usually `.h` files). +2. If there is a straight forward symbol difference between your kernel and the GKI + kernel, then do a `git blame` to find the commit. +3. Sometimes a symbol is deleted in a tree and you also want to delete it in + the other tree. To find the change that deleted the line, run this command + on the tree where the line was deleted: + + a. `git log -S "copy paste of deleted line/word" -- ` + + **NOTE:** Do not copy-paste tabs + + b. You will get a short list of commits. The first one is probably the one + you are looking for. Otherwise, go through the list until you find the + commit. + +4. Once you have identified the change, revert it in your kernel or [upload it + to ACK and get it merged](https://android.googlesource.com/kernel/common/+/987d0b5bcf096a478aaf96faf5a288b4c95e9d37/README.md). diff --git a/abi/abitool.py b/abi/abitool.py new file mode 100644 index 0000000000..b5b9a6e4d9 --- /dev/null +++ b/abi/abitool.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import re +import subprocess +import tempfile + +log = logging.getLogger(__name__) + + +def _collapse_impacted_interfaces(text): + """Removes impacted interfaces details, leaving just the summary count.""" + return re.sub( + r"^( *)([^ ]* impacted interfaces?):\n(?:^\1 .*\n)*", + r"\1\2\n", + text, + flags=re.MULTILINE) + + +def _collapse_offset_changes(text): + """Replaces "offset changed" lines with a one-line summary.""" + regex = re.compile( + r"^( *)('.*') offset changed from .* to .* \(in bits\) (\(by .* bits\))$") + items = [] + indent = "" + offset = "" + new_text = [] + + def emit_pending(): + if not items: + return + count = len(items) + if count == 1: + only = items[0] + line = "{}{} offset changed {}\n".format(indent, only, offset) + else: + first = items[0] + last = items[-1] + line = "{}{} ({} .. {}) offsets changed {}\n".format( + indent, count, first, last, offset) + del items[:] + new_text.append(line) + + for line in text.splitlines(True): + match = regex.match(line) + if match: + (new_indent, item, new_offset) = match.group(1, 2, 3) + if new_indent != indent or new_offset != offset: + emit_pending() + indent = new_indent + offset = new_offset + items.append(item) + else: + emit_pending() + new_text.append(line) + + emit_pending() + return "".join(new_text) + + +# TODO(b/157510812#comment15): Drop when libabigail fixed. +def _eliminate_spurious_blank_lines(text): + return re.sub( + r"^\n(^ CRC.*changed from [^ ]* to [^ ]*$)", + r"\1", + text, + flags=re.MULTILINE) + + +def _collapse_CRC_changes(text, limit): + """Preserves some CRC-only changes and summarises the rest. + + A CRC-only change is one like the following (indented and with a + trailing blank line). + + [C] 'function void* blah(type*)' at core.c:666:1 has some sub-type changes: + CRC value (modversions) changed from 0xf0f8820e to 0xe817181d + + Up to the first 'limit' changes will be emitted at the end of the + enclosing diff section. Any remaining ones will be summarised with + a line like the following. + + ... 17 omitted; 27 symbols have only CRC changes + + Args: + text: The report text. + limit: The maximum, integral number of CRC-only changes per diff section. + + Returns: + Updated report text. + """ + section_regex = re.compile(r"^[^ \n]") + change_regex = re.compile(r"^ \[C\] .*:$") + crc_regex = re.compile(r"^ CRC.*changed from [^ ]* to [^ ]*$") + blank_regex = re.compile(r"^$") + pending = [] + new_lines = [] + + def emit_pending(): + if not pending: + return + for (symbol_details, crc_details) in pending[0:limit]: + new_lines.extend([symbol_details, crc_details, "\n"]) + count = len(pending) + if count > limit: + new_lines.append(" ... {} omitted; {} symbols have only CRC changes\n\n" + .format(count - limit, count)) + pending.clear() + + lines = text.splitlines(True) + index = 0 + while index < len(lines): + line = lines[index] + if section_regex.match(line): + emit_pending() + if (index + 2 < len(lines) and change_regex.match(line) and + crc_regex.match(lines[index+1]) and blank_regex.match(lines[index+2])): + pending.append((line, lines[index+1])) + index += 3 + continue + new_lines.append(line) + index += 1 + + emit_pending() + return "".join(new_lines) + + +class AbiTool(object): + """ Base class for different kinds of abi analysis tools""" + def dump_kernel_abi(self, linux_tree, dump_path, symbol_list, + vmlinux_path=None, tidy=False): + raise NotImplementedError() + + def diff_abi(self, old_dump, new_dump, diff_report, short_report, symbol_list): + raise NotImplementedError() + + def name(self): + raise NotImplementedError() + +ABIDIFF_ERROR = (1<<0) +ABIDIFF_USAGE_ERROR = (1<<1) +ABIDIFF_ABI_CHANGE = (1<<2) +ABIDIFF_ABI_INCOMPATIBLE_CHANGE = (1<<3) + +class Libabigail(AbiTool): + """" Concrete AbiTool implementation for libabigail """ + def dump_kernel_abi(self, linux_tree, dump_path, symbol_list, + vmlinux_path=None, tidy=False): + with tempfile.NamedTemporaryFile() as temp_file: + temp_path = temp_file.name + + dump_abi_cmd = ['abidw', + # omit various sources of indeterministic abidw output + '--no-corpus-path', + '--no-comp-dir-path', + # use (more) stable type ids + '--type-id-style', + 'hash', + # the path containing vmlinux and *.ko + '--linux-tree', + linux_tree, + '--out-file', + temp_path] + + if vmlinux_path is not None: + dump_abi_cmd.extend(['--vmlinux', vmlinux_path]) + + if symbol_list is not None: + dump_abi_cmd.extend(['--kmi-whitelist', symbol_list]) + + subprocess.check_call(dump_abi_cmd) + + if tidy: + process_abi_command = ['abitidy', + '--all', + '--no-report-untyped', + '--input', temp_path, + '--output', dump_path] + else: + process_abi_command = ['cp', '--', temp_path, dump_path] + + subprocess.check_call(process_abi_command) + + def diff_abi(self, old_dump, new_dump, diff_report, short_report, + symbol_list, full_report): + log.info('libabigail diffing: {} and {} at {}'.format(old_dump, + new_dump, + diff_report)) + diff_abi_cmd = ['abidiff', + '--flag-indirect', + old_dump, + new_dump] + + if not full_report: + diff_abi_cmd.extend([ + '--leaf-changes-only', + '--impacted-interfaces', + ]) + + if symbol_list is not None: + diff_abi_cmd.extend(['--kmi-whitelist', symbol_list]) + + abi_changed = False + + with open(diff_report, 'w') as out: + try: + subprocess.check_call(diff_abi_cmd, stdout=out, stderr=out) + except subprocess.CalledProcessError as e: + if e.returncode & (ABIDIFF_ERROR | ABIDIFF_USAGE_ERROR): + raise + abi_changed = True # actual abi change + + if short_report is not None: + with open(diff_report) as full_report: + with open(short_report, 'w') as out: + text = full_report.read() + text = _collapse_impacted_interfaces(text) + text = _collapse_offset_changes(text) + text = _eliminate_spurious_blank_lines(text) + text = _collapse_CRC_changes(text, 3) + out.write(text) + + return abi_changed + +def get_abi_tool(abi_tool = "libabigail"): + if abi_tool == 'libabigail': + log.info('using libabigail for abi analysis') + return Libabigail() + + raise ValueError("not a valid abi_tool: %s" % abi_tool) diff --git a/abi/compare_to_symbol_list b/abi/compare_to_symbol_list new file mode 100755 index 0000000000..4a786f4607 --- /dev/null +++ b/abi/compare_to_symbol_list @@ -0,0 +1,52 @@ +#!/bin/bash + +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ "$#" -ne 2 ]; then + echo "ERROR: illegal number of parameters" >&2 + echo "Usage: compare_to_wl " >&2 + exit 1 +fi + +ksymtab=$(mktemp) +symbollist=$(mktemp) + +# List of objects, typically vmlinux + GKI modules +objects="${KMI_STRICT_MODE_OBJECTS:-vmlinux}" +objects=($objects) + +# Parse Module.symvers, and ignore non-exported and vendor-specific symbols +cut -d$'\t' -f2- $1 | while read line; do + ksym=($line) # 0=symbol name; 1=object name; 2=export type + if [[ "${ksym[2]}" != "EXPORT_SYMBOL"* ]]; then + continue + elif [[ ! " ${objects[@]} " =~ " ${ksym[1]} " ]]; then + continue + fi + echo "${ksym[0]}" +done | sort -u > $ksymtab + +sort -u "$2" > $symbollist + +delta=$(diff -u $ksymtab $symbollist | grep -v ' ') +rm $ksymtab $symbollist +if [ -n "$delta" ]; then + echo "ERROR: Differences between ksymtab and symbol list detected!" >&2 + echo "Symbols missing from ksymtab:" >&2 + echo "$delta" | sed -n -e 's/^+/ - /p' >&2 + echo "Symbols missing from symbol list:" >&2 + echo "$delta" | sed -n -e 's/^-/ - /p' >&2 + exit 1 +fi diff --git a/abi/diff_abi b/abi/diff_abi new file mode 100755 index 0000000000..1f4b4246e0 --- /dev/null +++ b/abi/diff_abi @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import argparse +import os +import sys + +from abitool import get_abi_tool + +def diff_abi(abitool, baseline, new, report, short_report, symbol_list, + full_report): + tool = get_abi_tool(abitool) + return tool.diff_abi(baseline, new, report, short_report, symbol_list, + full_report) + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--baseline', help='baseline dump to compare to', + required=True) + parser.add_argument('--new', help='new dump to compare to baseline', + required=True) + parser.add_argument('--abi-tool', default='libabigail', + help='abi tool to be used to monitor abi') + parser.add_argument('--report', help='where to write the report to', + required=True) + parser.add_argument('--short-report', help='where to write a short report to', + default=None) + parser.add_argument('--kmi-symbol-list', '--kmi-whitelist', default=None, + help='KMI symbol list to filter for') + parser.add_argument('--full-report', action='store_true', + help='Emit a detailled ABI report') + + args = parser.parse_args() + + abi_changed = diff_abi(args.abi_tool, + args.baseline, + args.new, + args.report, + args.short_report, + args.kmi_symbol_list, + args.full_report) + if abi_changed: + return 8 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/abi/dump_abi b/abi/dump_abi new file mode 100755 index 0000000000..7001e985be --- /dev/null +++ b/abi/dump_abi @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import argparse +import os +import sys + +from abitool import get_abi_tool + +def dump_abi(abitool, linux_tree, out_file, symbol_list, vmlinux_path=None, + tidy=False): + tool = get_abi_tool(abitool) + tool.dump_kernel_abi(linux_tree, out_file, symbol_list, vmlinux_path, tidy) + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--linux-tree', + help='Path to kernel tree containing ' + 'vmlinux and modules', + required=True) + parser.add_argument('--vmlinux', + help='Path to the vmlinux binary to consider to ' + 'emit the ABI of the union of vmlinux and its ' + 'modules', default=None) + parser.add_argument('--abi-tool', default='libabigail', + help='abi tool to be used to monitor abi') + parser.add_argument('--out-file', default=None, + help='where to write the ABI dump to') + parser.add_argument('--kmi-symbol-list', '--kmi-whitelist', default=None, + help='KMI symbol list to filter for') + parser.add_argument('--tidy', action='store_true', + help='postprocess ABI dump with abitidy') + + args = parser.parse_args() + + dump_abi(args.abi_tool, + args.linux_tree, + args.out_file or os.path.join(args.linux_tree, "abi.xml"), + args.kmi_symbol_list, + args.vmlinux, + args.tidy) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/abi/extract_symbols b/abi/extract_symbols new file mode 100755 index 0000000000..bb81e027cc --- /dev/null +++ b/abi/extract_symbols @@ -0,0 +1,352 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import argparse +import collections +import functools +import itertools +import os +import re +import subprocess +import sys + +_ALWAYS_INCLUDED = [ + "module_layout", # is exported even if CONFIG_TRIM_UNUSED_KSYMS is enabled + "__put_task_struct", # this allows us to keep `struct task_struct` stable +] +_ABIGAIL_HEADER = "[abi_symbol_list]" + +def symbol_sort(symbols): + # use the method that `sort` uses: case insensitive and ignoring + # underscores, that keeps symbols with related name close to each other. + # yeah, that is a bit brute force, but it gets the job done + + def __key(a): + """Creates a key for comparison of symbols.""" + # We want to sort underscore prefixed symbols along with those without, but + # before them. Hence add a trailing underscore for every missing leading + # one and strip all others. + # E.g. __blk_mq_end_request, _blk_mq_end_request, blk_mq_end_request get + # replaced by blkmqendrequest, blkmqendrequest_, blkmqendrequest__ and + # compared lexicographically. + + # if the caller passes None or an empty string something is odd, so assert + # and ignore if asserts are disabled as we do not need to deal with that + assert (a) + if not a: + return a + + tmp = a.lower() + for idx, c in enumerate(tmp): + if c != "_": + break + return (tmp.replace("_", "") + (5 - idx) * "_", a) + + return sorted(set(symbols), key=__key) + + +def find_binaries(directory): + """Locates vmlinux and kernel modules (*.ko).""" + vmlinux = None + modules = [] + for root, dirs, files in os.walk(directory): + for file in files: + if file.endswith(".ko"): + modules.append(os.path.join(root, file)) + elif file == "vmlinux": + vmlinux = os.path.join(root, file) + + return vmlinux, modules + + +def extract_undefined_symbols(modules): + """Extracts undefined symbols from a list of module files.""" + + # yes, we could pass all of them to nm, but I want to avoid hitting shell + # limits with long lists of modules + result = {} + for module in sorted(modules): + symbols = [] + out = subprocess.check_output(["llvm-nm", "--undefined-only", module], + stderr=subprocess.DEVNULL).decode("ascii") + for line in out.splitlines(): + symbols.append(line.strip().split()[1]) + + result[os.path.basename(module)] = symbol_sort(symbols) + + return result + + +def extract_exported_symbols(binary): + """Extracts the ksymtab exported symbols from a kernel binary.""" + symbols = [] + out = subprocess.check_output(["llvm-nm", "--defined-only", binary], + stderr=subprocess.DEVNULL).decode("ascii") + for line in out.splitlines(): + pos = line.find(" __ksymtab_") + if pos != -1: + symbols.append(line[pos + len(" __ksymtab_"):]) + + return symbol_sort(symbols) + +def extract_generic_exports(vmlinux, modules): + """Extracts the ksymtab exported symbols from vmlinux and a set of modules.""" + symbols = extract_exported_symbols(vmlinux) + for module in modules: + symbols.extend(extract_exported_symbols(module)) + return symbols + +def extract_exported_in_modules(modules): + """Extracts the ksymtab exported symbols for a list of kernel modules.""" + return {module: extract_exported_symbols(module) for module in modules} + + +def report_missing(module_symbols, exported): + """Reports missing symbols that are undefined, but not known in any binary.""" + for module, symbols in module_symbols.items(): + for symbol in symbols: + if symbol not in exported: + print("Symbol {} required by {} but not provided".format( + symbol, module)) + + +def add_dependent_symbols(module_symbols, exported): + """Checks the undefined symbols, and adds more to enforce missing dependencies.""" + for module, symbols in module_symbols.items(): + syms = [] + for symbol in symbols: + + # Tracepoints are exposed in the ABI using their matching struct + # tracepoint. Sadly this exposes callback functions as void * pointers, + # which make the ABI tooling ineffective to monitor tracepoint changes. + # To enable ABI checks covering tracepoint, add the matching __traceiter + # symbols to the symbol list as they are defined with full types. + if not symbol.startswith('__tracepoint_'): + continue + cur = symbol.replace('__tracepoint_', '__traceiter_') + if (cur not in exported) or (cur in symbols): + continue + syms.append(cur) + module_symbols[module].extend(syms) + + +def read_symbol_list(symbol_list): + """Reads a previously created libabigail symbol symbol list.""" + symbols = [] + with open(symbol_list) as wl: + for line in [l.strip() for l in wl]: + if not line or line.startswith("#") or line.startswith("["): + continue + symbols.append(line) + return symbols + + +def create_symbol_list(symbol_list, undefined_symbols, exported, + emit_module_symbol_lists, module_grouping, + additions_only): + """Creates a symbol symbol list for libabigail.""" + precious_symbols = set() + if additions_only: + precious_symbols.update(read_symbol_list(symbol_list)) + + symbol_counter = collections.Counter( + itertools.chain.from_iterable(undefined_symbols.values())) + + with open(symbol_list, "w") as wl: + + common_wl_section = symbol_sort([ + symbol for symbol, count in symbol_counter.items() + if (count > 1 or not module_grouping) and symbol in exported + ] + _ALWAYS_INCLUDED) + + # write the header + wl.write(_ABIGAIL_HEADER) + wl.write("\n") + if module_grouping: + wl.write("# commonly used symbols\n") + wl.write(" ") + wl.write("\n ".join(common_wl_section)) + wl.write("\n") + precious_symbols.difference_update(common_wl_section) + + for module, symbols in undefined_symbols.items(): + + if emit_module_symbol_lists: + mod_wl_file = symbol_list + "_" + os.path.splitext(module)[0] + with open(mod_wl_file, "w") as mod_wl: + # write the header + mod_wl.write(_ABIGAIL_HEADER) + mod_wl.write("\n ") + mod_wl.write("\n ".join([s for s in symbols if s in exported])) + mod_wl.write("\n") + + new_wl_section = symbol_sort([ + symbol for symbol in symbols + if symbol in exported and symbol not in common_wl_section + ]) + + if not new_wl_section: + continue + + wl.write("\n# required by {}\n ".format(module)) + wl.write("\n ".join(new_wl_section)) + wl.write("\n") + precious_symbols.difference_update(new_wl_section) + + if precious_symbols: + wl.write("\n# preserved by --additions-only\n ") + wl.write("\n ".join(symbol_sort(precious_symbols))) + wl.write("\n") + + +def main(): + """Extracts the required symbols for a directory full of kernel modules.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "directory", + nargs="?", + default=os.getcwd(), + help="the directory to search for kernel binaries") + + parser.add_argument( + "--skip-report-missing", + action="store_false", + dest="report_missing", + help="Do not report symbols required by modules, but missing from vmlinux" + ) + + parser.add_argument( + "--include-module-exports", + action="store_true", + help="Include inter-module symbols") + + parser.add_argument( + "--full-gki-abi", + action="store_true", + help="Assume all vmlinux and GKI module symbols are part of the ABI") + + parser.add_argument( + "--symbol-list", "--whitelist", + help="The symbol list to create") + + parser.add_argument( + "--additions-only", + action="store_true", + help="Read the existing symbol list and ensure no symbols get removed") + + parser.add_argument( + "--print-modules", + action="store_true", + help="Emit the names of the processed modules") + + parser.add_argument( + "--emit-module-symbol-lists", "--emit-module-whitelists", + action="store_true", + help="Emit a separate symbol list for each module") + + parser.add_argument( + "--skip-module-grouping", + action="store_false", + dest="module_grouping", + help="Do not group symbols by module.") + + parser.add_argument( + "--module-filter", + action="append", + dest="module_filters", + help="Only process modules matching the filter. Can be passed multiple times." + ) + + parser.add_argument( + "--gki-modules", + help="List of GKI modules which must be provided when the search directory contains both vendor and GKI modules") + + args = parser.parse_args() + + if not os.path.isdir(args.directory): + print("Expected a directory to search for binaries, but got %s" % + args.directory) + return 1 + + if args.emit_module_symbol_lists and not args.symbol_list: + print("Emitting module symbol lists requires the --symbol-list parameter.") + return 1 + + if args.symbol_list is None: + args.symbol_list = "/dev/stdout" + + # Locate the Kernel Binaries + vmlinux, modules = find_binaries(args.directory) + + if args.module_filters: + modules = [ + mod for mod in modules if any( + [re.search(f, os.path.basename(mod)) for f in args.module_filters]) + ] + + # Partition vendor and GKI modules in two lists + gki_modules = [] + if args.gki_modules is not None: + with open(args.gki_modules) as f: + gki_modules = [ os.path.basename(mod) for mod in f.read().splitlines() ] + gki_modules = [ mod for mod in modules if os.path.basename(mod) in gki_modules ] + modules = [ mod for mod in modules if mod not in gki_modules ] + + + if vmlinux is None or not os.path.isfile(vmlinux): + print("Could not find a suitable vmlinux file.") + return 1 + + # Get required symbols of all modules + undefined_symbols = extract_undefined_symbols(modules) + + # Get the actually defined and exported symbols + generic_exports = extract_generic_exports(vmlinux, gki_modules) + local_exports = extract_exported_in_modules(modules) + + # Build the list of all exported symbols (generic and local) + all_exported = list( + itertools.chain.from_iterable(local_exports.values())) + all_exported.extend(generic_exports) + all_exported = set(all_exported) + + add_dependent_symbols(undefined_symbols, all_exported) + + # For sanity, check for inconsistencies between required and exported symbols + # Do not do this analysis if module_filters are in place as likely + # inter-module dependencies are broken by this. + if args.report_missing and not args.module_filters: + report_missing(undefined_symbols, all_exported) + + # If specified, create the symbol list + if args.symbol_list: + create_symbol_list( + args.symbol_list, + { "full-gki-abi": generic_exports } if args.full_gki_abi else undefined_symbols, + all_exported if args.include_module_exports else generic_exports, + args.emit_module_symbol_lists, + args.module_grouping, + args.additions_only) + + if args.print_modules: + print("These modules have been considered when creating the symbol list:") + print(" " + + "\n ".join(sorted([os.path.basename(mod) for mod in modules]))) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/abi/flatten_symbol_list b/abi/flatten_symbol_list new file mode 100755 index 0000000000..3a1975f917 --- /dev/null +++ b/abi/flatten_symbol_list @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import configparser +import sys + +def main(): + """Convert a KMI symbol list in libabigail format to a raw list.""" + if sys.stdin.isatty(): + print("ERROR: missing KMI symbol list on the standard input") + return 1 + + sl = configparser.ConfigParser(allow_no_value=True, strict=False) + sl.optionxform = str + sl.read_file(sys.stdin) + + ksyms = set() + for section in (s for s in sl.sections() if s.endswith(('whitelist', + 'symbol_list'))): + ksyms.update(sl[section]) + print('\n'.join(sorted(ksyms))) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/abi/gki_check b/abi/gki_check new file mode 100755 index 0000000000..3ac5941f9c --- /dev/null +++ b/abi/gki_check @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import argparse +import base64 +import json +import logging +import os +import subprocess +import sys +import urllib.request + +from abitool import get_abi_tool + +_BASE_URL_ABI_XML = ('https://android.googlesource.com/kernel/common/' + '+/refs/heads/android-{}/abi_gki_aarch64.xml?format={}') + +logger = logging.getLogger('gki_check') +logging.basicConfig(level=logging.INFO, format='%(message)s') + + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument( + '--kernel-version', + help='The Kernel version to compare against', + choices=['4.19', '5.4'], + default='5.4') + + parser.add_argument( + '--linux-tree', + help='Path to kernel tree containing ' + 'vmlinux and modules', + required=True) + + parser.add_argument( + '--abi-tool', + default='libabigail', + help='tool to monitor the ABI') + + parser.add_argument( + '--kmi-symbol-list', '--kmi-whitelist', + default=None, + help='KMI symbol list to filter for') + + args = parser.parse_args() + + abi_tool = get_abi_tool(args.abi_tool) + + # We assume the --linux-tree to be our working directory, hence download all + # the stuff there. + + gki_check_dir = os.path.join(args.linux_tree, '.gki_check') + logger.info('Creating working directory %s ...', gki_check_dir) + os.makedirs(gki_check_dir, exist_ok=True) + + upstream_xml = os.path.join(gki_check_dir, 'upstream.xml') + local_xml = os.path.join(gki_check_dir, 'local.xml') + abi_report = os.path.join(gki_check_dir, 'abi.report') + abi_report_short = os.path.join(gki_check_dir, 'abi.report.short') + + # Download GKI abi.xml from either 5.4 (default) or 4.19 + abi_xml_url = _BASE_URL_ABI_XML.format(args.kernel_version, 'TEXT') + + base_is_current = False + if os.path.exists(upstream_xml): + current_hash = subprocess.check_output(['git', 'hash-object', + upstream_xml]).strip().decode() + + response = urllib.request.urlopen( + _BASE_URL_ABI_XML.format(args.kernel_version, 'JSON')) + data = response.read().decode('utf-8') + new_hash = json.loads(data[data.index('{'):])['id'] + + if current_hash == new_hash: + base_is_current = True + logger.info('ABI representation is already up-to-date. ' + 'Skipping download ...') + + if not base_is_current: + logger.info('Downloading ABI representation for android-%s ...', + args.kernel_version) + + response = urllib.request.urlopen(abi_xml_url) + data = response.read() + with open(upstream_xml, 'wb') as reference_xml: + reference_xml.write(base64.b64decode(data)) + + logger.info('Extracting ABI information from your local build ...') + abi_tool.dump_kernel_abi(args.linux_tree, local_xml, args.kmi_symbol_list) + + logger.info('Comparing ABI information to reference ...') + abi_tool.diff_abi(local_xml, upstream_xml, abi_report, abi_report_short, + args.kmi_symbol_list) + + logger.info('ABI Analysis complete, please find the result below') + with open(abi_report_short) as short_report: + print(short_report.read()) + + logger.info('Upstream %s', upstream_xml) + logger.info('Local representation: %s', local_xml) + logger.info('Full report: %s', abi_report) + logger.info('Short report: %s', abi_report_short) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/abi/kmi_defines.py b/abi/kmi_defines.py new file mode 100755 index 0000000000..07bf17faa3 --- /dev/null +++ b/abi/kmi_defines.py @@ -0,0 +1,787 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""kmi_defines extract #define compile time constants from a Linux build. + +The kmi_defines tool is used to examine the output of a Linux build +and extract from it C #define statements that define compile time +constant expressions for the purpose of tracking them as part of the +KMI (Kernel Module Interface) so that changes to their values can be +prevented so as to ensure a constant KMI for kernel modules for the +AOSP GKI Linux kernel project. + +This code is python3 only, it does not require any from __future__ +imports. This is a standalone program, it is not meant to be used as +a module by other programs. + +This program runs under the multiprocessing module. Work done within +a multiprocessing.Pool does not perform error logging or affects any +state other than the value that it computes and returns via the function +mapped through the pool's map() function. The reason that no external +state is affected (for example error loggiing) is to avoid to have to +even think about what concurrent updates would cause to shuch a facility. +""" + +# TODO(pantin): per Matthias review feedback: "drop the .py from the +# filename after(!) the review has completed. As last action. Until +# then we can have the syntax highlighting here in Gerrit." + +import argparse +import collections +import logging +import multiprocessing +import os +import pathlib +import re +import subprocess +import sys +from typing import List, Optional, Tuple +from typing import Set # pytype needs this, pylint: disable=unused-import + +INDENT = 4 # number of spaces to indent for each depth level +COMPILER = "clang" # TODO(pantin): should be determined at run-time + +# Dependency that is hidden by the transformation of the .o.d file into +# the .o.cmd file as part of the Linux build environment. This header is +# purposely removed and replaced by fictitious set of empty header files +# that were never part of the actual compilation of the .o files. Those +# fictitious empty files are generated under the build environment output +# directory in this subdirectory: +# include/config +# +# This is the actual header file that was part of the compilation of every +# .o file, the HIDDEN_DEP are added to the dependencies of every .o file. +# +# It is important that this file be added because it is unknowable whether +# the #defines in it were depended upon by a module to alter its behaviour +# at compile time. For example to pass some flags or not pass some flags +# to a function. + +HIDDEN_DEP = "include/generated/autoconf.h" + + +class StopError(Exception): + """Exception raised to stop work when an unexpected error occurs.""" + + +def dump(this) -> None: + """Dump the data in this. + + This is for debugging purposes, it does not handle every type, only + the types used by the underlying code are handled. This will not be + part of the final code, or if it is, it will be significantly enhanced + or replaced by some other introspection mechanism to serialize data. + """ + def dump_this(this, name: str, depth: int) -> None: + """Dump the data in this.""" + if name: + name += " = " + if isinstance(this, str): + indent = " " * (depth * INDENT) + print(indent + name + this) + elif isinstance(this, bool): + indent = " " * (depth * INDENT) + print(indent + name + str(this)) + elif isinstance(this, List): + dump_list(this, name, depth) + elif isinstance(this, Set): + dump_set(this, name, depth) + else: + dump_object(this, name, depth) + + def dump_list(lst: List[str], name: str, depth: int) -> None: + """Dump the data in lst.""" + indent = " " * (depth * INDENT) + print(indent + name + "{") + index = 0 + for entry in lst: + dump_this(entry, f"[{index}]", depth + 1) + index += 1 + print(indent + "}") + + def dump_set(aset: Set[str], name: str, depth: int) -> None: + """Dump the data in aset.""" + lst = list(aset) + lst.sort() + dump_list(lst, name, depth) + + def dump_object(this, name: str, depth: int) -> None: + """Dump the data in this.""" + indent = " " * (depth * INDENT) + print(indent + name + + re.sub(r"(^$)", "", str(type(this))) + " {") + for key, val in this.__dict__.items(): + dump_this(val, key, depth + 1) + print(indent + "}") + + dump_this(this, "", 0) + + +def readfile(name: str) -> str: + """Open a file and return its contents in a string as its value.""" + try: + with open(name) as file: + return file.read() + except OSError as os_error: + raise StopError("readfile() failed for: " + name + "\n" + "original OSError: " + str(os_error.args)) + + +def file_must_exist(file: str) -> None: + """If file is invalid print raise a StopError.""" + if not os.path.exists(file): + raise StopError("file does not exist: " + file) + if not os.path.isfile(file): + raise StopError("file is not a regular file: " + file) + + +def makefile_depends_get_dependencies(depends: str) -> List[str]: + """Return list with the dependencies of a makefile target. + + Split the makefile depends specification, the name of the dependent is + followed by ":" its dependencies follow the ":". There could be spaces + around the ":". Line continuation characters, i.e. "\" are consumed by + the regular expression that splits the specification. + + This results in a list with the dependent first, and its dependencies + in the remainder of the list, return everything in the list other than + the first element. + """ + return re.split(r"[:\s\\]+", re.sub(r"[\s\\]*\Z", "", depends))[1:] + + +def makefile_assignment_split(assignment: str) -> Tuple[str, str]: + """Split left:=right into a tuple with the left and right parts. + + Spaces around the := are also removed. + """ + result = re.split(r"\s*:=\s*", assignment, maxsplit=1) + if len(result) != 2: + raise StopError( + "expected: 'left:=right' in: " + + assignment) + return result[0], result[1] # left, right + + +def get_src_ccline_deps(obj: str) -> Optional[Tuple[str, str, List[str]]]: + """Get the C source file, its cc_line, and non C source dependencies. + + If the tool used to produce the object is not the compiler, or if the + source file is not a C source file None is returned. + + Otherwise it returns a triplet with the C source file name, its cc_line, + the remaining dependencies. + """ + o_cmd = os.path.join(os.path.dirname(obj), + "." + os.path.basename(obj) + ".cmd") + + contents = readfile(o_cmd) + contents = re.sub(r"\$\(wildcard[^)]*\)", " ", contents) + contents = re.sub(r"[ \t]*\\\n[ \t]*", " ", contents) + lines = lines_to_list(contents) + + cc_line = None + deps = None + source = None + for line in lines: + if line.startswith("cmd_"): + cc_line = line + elif line.startswith("deps_"): + deps = line + elif line.startswith("source_"): + source = line + + if cc_line is None: + raise StopError("missing cmd_* variable in: " + o_cmd) + _, cc_line = makefile_assignment_split(cc_line) + if cc_line.split(maxsplit=1)[0] != COMPILER: + # The object file was made by strip, symbol renames, etc. + # i.e. it was not the result of running the compiler, thus + # it can not contribute to #define compile time constants. + return None + + if source is None: + raise StopError("missing source_* variable in: " + o_cmd) + _, source = makefile_assignment_split(source) + source = source.strip() + if not source.endswith(".c"): + return None + + if deps is None: + raise StopError("missing deps_* variable in: " + o_cmd) + _, deps = makefile_assignment_split(deps) + dependendencies = deps.split() + dependendencies.append(HIDDEN_DEP) + + return source, cc_line, dependendencies + + +def lines_to_list(lines: str) -> List[str]: + """Split a string into a list of non-empty lines.""" + return [line for line in lines.strip().splitlines() if line] + + +def lines_get_first_line(lines: str) -> str: + """Return the first non-empty line in lines.""" + return lines.strip().splitlines()[0] + + +def shell_line_to_o_files_list(line: str) -> List[str]: + """Return a list of .o files in the files list.""" + return [entry for entry in line.split() if entry.endswith(".o")] + + +def run(args: List[str], + raise_on_failure: bool = True) -> subprocess.CompletedProcess: + """Run the program specified in args[0] with the arguments in args[1:].""" + try: + # This argument does not always work for subprocess.run() below: + # check=False + # neither that nor: + # check=True + # prevents an exception from being raised if the program that + # will be executed is not found + + completion = subprocess.run(args, capture_output=True, text=True) + if completion.returncode != 0 and raise_on_failure: + raise StopError("execution failed for: " + " ".join(args)) + return completion + except OSError as os_error: + raise StopError("failure executing: " + " ".join(args) + "\n" + "original OSError: " + str(os_error.args)) + + +class KernelModule: + """A kernel module, i.e. a *.ko file.""" + def __init__(self, kofile: str) -> None: + """Construct a KernelModule object.""" + # An example argument is used below, assuming kofile is: + # possibly/empty/dirs/modname.ko + # + # Meant to refer to this module, shown here relative to the top of + # the build directory: + # drivers/usb/gadget/udc/modname.ko + # the values assigned to the members are shown in the comments below. + + self._file = os.path.realpath(kofile) # /abs/dirs/modname.ko + self._base = os.path.basename(self._file) # modname.ko + self._directory = os.path.dirname(self._file) # /abs/dirs + self._cmd_file = os.path.join(self._directory, + "." + self._base + ".cmd") + self._cmd_text = readfile(self._cmd_file) + + # Some builds append a '; true' to the .modname.ko.cmd, remove it + + self._cmd_text = re.sub(r";\s*true\s*$", "", self._cmd_text) + + # The modules .modname.ko.cmd file contains a makefile snippet, + # for example: + # cmd_drivers/usb/gadget/udc/dummy_hcd.ko := ld.lld -r ... + # + # Split the string prior to the spaces followed by ":=", and get + # the first element of the resulting list. If the string was not + # split (because it did not contain a ":=" then the input string + # is returned, by the re.sub() below, as the only element of the list. + + left, _ = makefile_assignment_split(self._cmd_text) + self._rel_file = re.sub(r"^cmd_", "", left) + if self._rel_file == left: + raise StopError("expected: 'cmd_' at start of content of: " + + self._cmd_file) + + base = os.path.basename(self._rel_file) + if base != self._base: + raise StopError("module name mismatch: " + base + " vs " + + self._base) + + self._rel_dir = os.path.dirname(self._rel_file) + + # The final step in the build of kernel modules is based on two .o + # files, one with the module name followed by .o and another followed + # by .mod.o + # + # The following test verifies that assumption, in case a module is + # built differently in the future. + # + # Even when there are multiple source files, the .o files that result + # from compiling them are all linked into a single .o file through an + # intermediate link step, that .o files is named: + # os.path.join(self._rel_dir, kofile_name + ".o") + + kofile_name, _ = os.path.splitext(self._base) + objs = shell_line_to_o_files_list(self._cmd_text) + objs.sort() + expected = [ # sorted, i.e.: .mod.o < .o + os.path.join(self._rel_dir, kofile_name + ".mod.o"), + os.path.join(self._rel_dir, kofile_name + ".o") + ] + if objs != expected: + raise StopError("unexpected .o files in: " + self._cmd_file) + + def get_build_dir(self) -> str: + """Return the top level build directory. + + I.e. the directory where the output of the Linux build is stored. + + Note that this, like pretty much all the code, can raise an exception, + by construction, if an exception is raised while an object is being + constructed, or after it is constructed, the object will not be used + thereafter (at least not any object explicitly created by this + program). Many other places, for example the ones that call readfile() + can raise exceptions, the code is located where it belongs. + + In this specific case, the computation of index, and the derived + invariant that it be >= 0, is predicated by the condition checked + below, if the exception is not raised, then index is >= 0. + """ + if not self._file.endswith(self._rel_file): + raise StopError("could not find: " + self._rel_file + + " at end of: " + self._file) + index = len(self._file) - len(self._rel_file) + if index > 0 and self._file[index - 1] == os.sep: + index -= 1 + build_dir = self._file[0:index] + return build_dir + + def get_object_files(self, build_dir: str) -> List[str]: + """Return a list object files that used to link the kernel module. + + The ocmd_file is the file with extension ".o.cmd" (see below). + If the ocmd_file has a more than one line in it, its because the + module is made of a single source file and the ocmd_file has the + compilation rule and dependencies to build it. If it has a single + line single line it is because it builds the .o file by linking + multiple .o files. + """ + + kofile_name, _ = os.path.splitext(self._base) + ocmd_file = os.path.join(build_dir, self._rel_dir, + "." + kofile_name + ".o.cmd") + ocmd_content = readfile(ocmd_file) + + olines = lines_to_list(ocmd_content) + if len(olines) > 1: # module made from a single .o file + return [os.path.join(build_dir, self._rel_dir, kofile_name + ".o")] + + # Multiple .o files in the module + + _, ldline = makefile_assignment_split(olines[0]) + return [ + os.path.realpath(os.path.join(build_dir, obj)) + for obj in shell_line_to_o_files_list(ldline) + ] + + +class Kernel: + """The Linux kernel component itself, i.e. vmlinux.o.""" + def __init__(self, kernel: str) -> None: + """Construct a Kernel object.""" + self._kernel = os.path.realpath(kernel) + self._build_dir = os.path.dirname(self._kernel) + libs = os.path.join(self._build_dir, "vmlinux.libs") + objs = os.path.join(self._build_dir, "vmlinux.objs") + file_must_exist(libs) + file_must_exist(objs) + contents = readfile(libs) + archives_and_objects = contents.split() + contents = readfile(objs) + archives_and_objects += contents.split() + self._archives_and_objects = [(os.path.join(self._build_dir, file) + if not os.path.isabs(file) else file) + for file in archives_and_objects] + + def get_build_dir(self) -> str: + """Return the top level build directory. + + I.e. the directory where the output of the Linux build is stored. + """ + return self._build_dir + + def get_object_files(self, build_dir: str) -> List[str]: + """Return a list object files that where used to link the kernel.""" + olist = [] + for file in self._archives_and_objects: + if file.endswith(".o"): + if not os.path.isabs(file): + file = os.path.join(build_dir, file) + olist.append(os.path.realpath(file)) + continue + + if not file.endswith(".a"): + raise StopError("unknown file type: " + file) + + completion = run(["ar", "t", file]) + objs = lines_to_list(completion.stdout) + + for obj in objs: + if not os.path.isabs(obj): + obj = os.path.join(build_dir, obj) + olist.append(os.path.realpath(obj)) + + return olist + + +class Target: # pylint: disable=too-few-public-methods + """Target of build and the information used to build it.""" + + # The compiler invocation has this form: + # clang -Wp,-MD,file.o.d ... -c -o file.o file.c + # these constants reflect that knowledge in the code, e.g.: + # - the "-Wp,_MD,file.o.d" is at WP_MD_FLAG_INDEX + # - the "-c" is at index C_FLAG_INDEX + # - the "-o" is at index O_FLAG_INDEX + # - the "file.o" is at index OBJ_INDEX + # - the "file.c" is at index SRC_INDEX + # + # There must be at least MIN_CC_LIST_LEN options in that command line. + # This knowledge is verified at run time in __init__(), see comments + # there. + + MIN_CC_LIST_LEN = 6 + WP_MD_FLAG_INDEX = 1 + C_FLAG_INDEX = -4 + O_FLAG_INDEX = -3 + OBJ_INDEX = -2 + SRC_INDEX = -1 + + def __init__(self, obj: str, src: str, cc_line: str, + deps: List[str]) -> None: + self._obj = obj + self._src = src + self._deps = deps + + # The cc_line, eventually slightly modified, will be used to run + # the compiler in various ways. The cc_line could be fed through + # the shell to deal with the single-quotes in the cc_line that are + # there to quote the double-quotes meant to be part of a C string + # literal. Specifically, this occurs in to pass KBUILD_MODNAME and + # KBUILD_BASENAME, for example: + # -DKBUILD_MODNAME='"aes_ce_cipher"' + # -DKBUILD_BASENAME='"aes_cipher_glue"' + # + # Causing an extra execve(2) of the shell, just to deal with a few + # quotes is wasteful, so instead, here the quotes, in this specific + # case are removed. This can be done, easiest just by removing the + # single quotes with: + # cc_cmd = re.sub(r"'", "", cc_line) + # + # But this could mess up other quote usage in the future, for example + # using double quotes or backslash to quote a single quote meant to + # actually be seen by the compiler. + # + # As an alternative, and for this to be more robust, the specific + # cases that are known, i.e. the two -D shown above, are dealt with + # individually and if there are any single or double quotes, or + # backslashes the underlying work is stopped. + # + # Note that the cc_line comes from the .foo.o.cmd file which is a + # makefile snippet, so the actual syntax there is also subject to + # whatever other things make would want to do with them. Instead + # of doing the absolutely correct thing, which would actually be + # to run this through make to have make run then through the shell + # this program already has knowledge about these .cmd files and how + # they are formed. This compromise, or coupling of knowledge, is a + # source of fragility, but not expected to cause much trouble in the + # future as the Linux build evolves. + + cc_cmd = re.sub( + r"""-D(KBUILD_BASENAME|KBUILD_MODNAME)='("[a-zA-Z0-9_.:]*")'""", + r"-D\1=\2", cc_line) + cc_list = cc_cmd.split() + + # TODO(pantin): the handling of -D... arguments above is done better + # in a later commit by using shlex.split(). Please ignore for now. + # TODO(pantin): possibly use ArgumentParser to make this more robust. + + # The compiler invocation has this form: + # clang -Wp,-MD,file.o.d ... -c -o file.o file.c + # + # The following checks are here to ensure that if this assumption is + # broken, failures occur. The indexes *_INDEX are hardcoded, they + # could in principle be determined at run time, the -o argument could + # be in a future update to the Linux build could changed to be a + # single argument with the object file name (as in: -ofile.o) which + # could also be detected in code at a later time. + + if (len(cc_list) < Target.MIN_CC_LIST_LEN + or not cc_list[Target.WP_MD_FLAG_INDEX].startswith("-Wp,-MD,") + or cc_list[Target.C_FLAG_INDEX] != "-c" + or cc_list[Target.O_FLAG_INDEX] != "-o"): + raise StopError("unexpected or missing arguments for: " + obj + + " cc_line: " + cc_line) + + # Instead of blindly normalizing the source and object arguments, + # they are only normalized if that allows the expected invariants + # to be verified, otherwise they are left undisturbed. Note that + # os.path.normpath() does not turn relative paths into absolute + # paths, it just removes up-down walks (e.g. a/b/../c -> a/c). + + def verify_file(file: str, index: int, kind: str, cc_list: List[str], + target_file: str) -> None: + file_in_cc_list = cc_list[index] + if not file.endswith(file_in_cc_list): + file_normalized = os.path.normpath(file_in_cc_list) + if not file.endswith(file_normalized): + raise StopError(f"unexpected {kind} argument for: " + f"{target_file} value was: " + f"{file_in_cc_list}") + cc_list[index] = file_normalized + + verify_file(obj, Target.OBJ_INDEX, "object", cc_list, obj) + verify_file(src, Target.SRC_INDEX, "source", cc_list, obj) + + self._cc_list = cc_list + + +class KernelComponentBase: # pylint: disable=too-few-public-methods + """Base class for KernelComponentCreationError and KernelComponent. + + There is not much purpose for this class other than to satisfy the strong + typing checks of pytype, with looser typing, this could be removed but at + the risk of invoking member functions at run-time on objects that do not + provide them. Having this class makes the code more reliable. + """ + def get_error(self) -> Optional[str]: # pylint: disable=no-self-use + """Return None for the error, means there was no error.""" + return None + + def get_deps_set(self) -> Set[str]: # pylint: disable=no-self-use + """Return the set of dependencies for the kernel component.""" + return set() + + def is_kernel(self) -> bool: # pylint: disable=no-self-use + """Is this the kernel?""" + return False + + +class KernelComponentCreationError(KernelComponentBase): # pylint: disable=too-few-public-methods + """A KernelComponent creation error. + + When a KernelComponent creation fails, or the creation of its subordinate + Kernel or KernelModule creation fails, a KernelComponentCreationError + object is created to store the information relevant to the failure. + """ + def __init__(self, filename: str, error: str) -> None: + """Construct a KernelComponentCreationError object.""" + self._error = error + self._filename = filename + + def get_error(self) -> Optional[str]: + """Return the error.""" + return self._filename + ": " + self._error + + +class KernelComponent(KernelComponentBase): + """A kernel component, either vmlinux.o or a *.ko file. + + Inspect a Linux kernel module (a *.ko file) or the Linux kernel to + determine what was used to build it: object filess, source files, header + files, and other information that is produced as a by-product of its build. + """ + def __init__(self, filename: str) -> None: + """Construct a KernelComponent object.""" + if filename.endswith("vmlinux.o"): + self._kernel = True + self._kind = Kernel(filename) + else: + self._kernel = False + self._kind = KernelModule(filename) + self._build_dir = self._kind.get_build_dir() + self._source_dir = self._get_source_dir() + self._files_o = self._kind.get_object_files(self._build_dir) + self._files_o.sort() + + # using a set because there is no unique flag to list.sort() + deps_set = set() + + self._targets = [] + for obj in self._files_o: + file_must_exist(obj) + result = get_src_ccline_deps(obj) + if result is None: + continue + src, cc_line, dependendencies = result + + file_must_exist(src) + depends = [] + for dep in dependendencies: + if not os.path.isabs(dep): + dep = os.path.join(self._build_dir, dep) + dep = os.path.realpath(dep) + depends.append(dep) + deps_set.add(dep) + + if not os.path.isabs(src): + src = os.path.join(self._build_dir, src) + src = os.path.realpath(src) + self._targets.append(Target(obj, src, cc_line, depends)) + + for dep in [dep for dep in list(deps_set) if not dep.endswith(".h")]: + deps_set.remove(dep) + self._deps_set = deps_set + + def _get_source_dir(self) -> str: + """Return the top level Linux kernel source directory.""" + source = os.path.join(self._build_dir, "source") + if not os.path.islink(source): + raise StopError("could not find source symlink: " + source) + + if not os.path.isdir(source): + raise StopError("source symlink not a directory: " + source) + + source_dir = os.path.realpath(source) + if not os.path.isdir(source_dir): + raise StopError("source directory not a directory: " + source_dir) + + return source_dir + + def get_deps_set(self) -> Set[str]: + """Return the set of dependencies for the kernel component.""" + return self._deps_set + + def is_kernel(self) -> bool: + """Is this the kernel?""" + return self._kernel + + +def kernel_component_factory(filename: str) -> KernelComponentBase: + """Make an InfoKmod or an InfoKernel object for file and return it.""" + try: + return KernelComponent(filename) + except StopError as stop_error: + return KernelComponentCreationError(filename, + " ".join([*stop_error.args])) + + +class KernelComponentProcess(multiprocessing.Process): + """Process to make the KernelComponent concurrently.""" + def __init__(self) -> None: + multiprocessing.Process.__init__(self) + self._queue = multiprocessing.Queue() + self.start() + + def run(self) -> None: + """Create and save the KernelComponent.""" + self._queue.put(kernel_component_factory("vmlinux.o")) + + def get_component(self) -> KernelComponentBase: + """Return the kernel component.""" + kernel_component = self._queue.get() + self.join() # must be after queue.get() otherwise it deadlocks + return kernel_component + + +def work_on_all_components(options) -> List[KernelComponentBase]: + """Return a list of KernelComponentBase objects.""" + files = [str(ko) for ko in pathlib.Path().rglob("*.ko")] + if options.sequential: + return [ + kernel_component_factory(file) for file in ["vmlinux.o"] + files + ] + + # There is significantly more work to be done for the vmlinux.o than + # the *.ko kernel modules. A dedicated process is started to do the + # work for vmlinux.o as soon as possible instead of leaving it to the + # vagaries of multiprocessing.Pool() and how it would spreads the work. + # This significantly reduces the elapsed time for this work. + + kernel_component_process = KernelComponentProcess() + + chunk_size = 128 + processes = max(1, len(files) // (chunk_size * 3)) + processes = min(processes, os.cpu_count()) + with multiprocessing.Pool(processes) as pool: + components = pool.map(kernel_component_factory, files, chunk_size) + + kernel_component = kernel_component_process.get_component() + + return [kernel_component] + components + + +def work_on_whole_build(options) -> int: + """Work on the whole build to extract the #define constants.""" + if not os.path.isfile("vmlinux.o"): + logging.error("file not found: vmlinux.o") + return 1 + components = work_on_all_components(options) + failed = False + header_count = collections.defaultdict(int) + for comp in components: + error = comp.get_error() + if error: + logging.error(error) + failed = True + continue + deps_set = comp.get_deps_set() + for header in deps_set: + header_count[header] += 1 + if failed: + return 1 + if options.dump: + dump(components) + if options.dump and options.includes: + print() + if options.includes: + for header, count in header_count.items(): + if count >= 2: + print(header) + return 0 + + +def main() -> int: + """Extract #define compile time constants from a Linux build.""" + def existing_file(file): + if not os.path.isfile(file): + raise argparse.ArgumentTypeError( + "{0} is not a valid file".format(file)) + return file + + parser = argparse.ArgumentParser() + parser.add_argument("-d", + "--dump", + action="store_true", + help="dump internal state") + parser.add_argument("-s", + "--sequential", + action="store_true", + help="execute without concurrency") + group = parser.add_mutually_exclusive_group() + group.add_argument("-i", + "--includes", + action="store_true", + help="show relevant include files") + group.add_argument("-c", + "--component", + type=existing_file, + help="show information for a component") + options = parser.parse_args() + + if not options.component: + return work_on_whole_build(options) + + comp = kernel_component_factory(options.component) + + error = comp.get_error() + if error: + logging.error(error) + return 1 + if options.dump: + dump([comp]) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/build-tools/path/linux-x86/abidiff b/build-tools/path/linux-x86/abidiff new file mode 120000 index 0000000000..498d3cf49f --- /dev/null +++ b/build-tools/path/linux-x86/abidiff @@ -0,0 +1 @@ +../../../../prebuilts/kernel-build-tools/linux-x86/bin/abidiff \ No newline at end of file diff --git a/build-tools/path/linux-x86/abidw b/build-tools/path/linux-x86/abidw new file mode 120000 index 0000000000..a2f937ef48 --- /dev/null +++ b/build-tools/path/linux-x86/abidw @@ -0,0 +1 @@ +../../../../prebuilts/kernel-build-tools/linux-x86/bin/abidw \ No newline at end of file diff --git a/build-tools/path/linux-x86/abitidy b/build-tools/path/linux-x86/abitidy new file mode 120000 index 0000000000..384f0cc886 --- /dev/null +++ b/build-tools/path/linux-x86/abitidy @@ -0,0 +1 @@ +../../../../prebuilts/kernel-build-tools/linux-x86/bin/abitidy \ No newline at end of file diff --git a/build-tools/path/linux-x86/avbtool b/build-tools/path/linux-x86/avbtool new file mode 120000 index 0000000000..8a69eb5751 --- /dev/null +++ b/build-tools/path/linux-x86/avbtool @@ -0,0 +1 @@ +../../../../prebuilts/kernel-build-tools/linux-x86/bin/avbtool \ No newline at end of file diff --git a/build-tools/path/linux-x86/awk b/build-tools/path/linux-x86/awk new file mode 120000 index 0000000000..b3ad24f251 --- /dev/null +++ b/build-tools/path/linux-x86/awk @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/awk \ No newline at end of file diff --git a/build-tools/path/linux-x86/basename b/build-tools/path/linux-x86/basename new file mode 120000 index 0000000000..451ccc99a0 --- /dev/null +++ b/build-tools/path/linux-x86/basename @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/basename \ No newline at end of file diff --git a/build-tools/path/linux-x86/bc b/build-tools/path/linux-x86/bc new file mode 120000 index 0000000000..7616002493 --- /dev/null +++ b/build-tools/path/linux-x86/bc @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/bc \ No newline at end of file diff --git a/build-tools/path/linux-x86/bison b/build-tools/path/linux-x86/bison new file mode 100755 index 0000000000..2edbffe141 --- /dev/null +++ b/build-tools/path/linux-x86/bison @@ -0,0 +1,7 @@ +#!/bin/sh + +base=$(readlink -f $(dirname $0)) +export BISON_PKGDATADIR=$(readlink -f ${base}/../../../../prebuilts/build-tools/common/bison) +export M4=${base}/m4 + +${base}/../../../../prebuilts/build-tools/linux-x86/bin/bison $* diff --git a/build-tools/path/linux-x86/blk_alloc_to_base_fs b/build-tools/path/linux-x86/blk_alloc_to_base_fs new file mode 120000 index 0000000000..41c4f336c2 --- /dev/null +++ b/build-tools/path/linux-x86/blk_alloc_to_base_fs @@ -0,0 +1 @@ +../../../../prebuilts/kernel-build-tools/linux-x86/bin/blk_alloc_to_base_fs \ No newline at end of file diff --git a/build-tools/path/linux-x86/build_image b/build-tools/path/linux-x86/build_image new file mode 120000 index 0000000000..1b90a2b232 --- /dev/null +++ b/build-tools/path/linux-x86/build_image @@ -0,0 +1 @@ +../../../../prebuilts/kernel-build-tools/linux-x86/bin/build_image \ No newline at end of file diff --git a/build-tools/path/linux-x86/build_super_image b/build-tools/path/linux-x86/build_super_image new file mode 120000 index 0000000000..694127aa35 --- /dev/null +++ b/build-tools/path/linux-x86/build_super_image @@ -0,0 +1 @@ +../../../../prebuilts/kernel-build-tools/linux-x86/bin/build_super_image \ No newline at end of file diff --git a/build-tools/path/linux-x86/bzcat b/build-tools/path/linux-x86/bzcat new file mode 120000 index 0000000000..63dbd42ee5 --- /dev/null +++ b/build-tools/path/linux-x86/bzcat @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/bzcat \ No newline at end of file diff --git a/build-tools/path/linux-x86/bzip2 b/build-tools/path/linux-x86/bzip2 new file mode 120000 index 0000000000..4b121ad455 --- /dev/null +++ b/build-tools/path/linux-x86/bzip2 @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/bzip2 \ No newline at end of file diff --git a/build-tools/path/linux-x86/cat b/build-tools/path/linux-x86/cat new file mode 120000 index 0000000000..451e54fe81 --- /dev/null +++ b/build-tools/path/linux-x86/cat @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/cat \ No newline at end of file diff --git a/build-tools/path/linux-x86/chmod b/build-tools/path/linux-x86/chmod new file mode 120000 index 0000000000..7a46fb3ded --- /dev/null +++ b/build-tools/path/linux-x86/chmod @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/chmod \ No newline at end of file diff --git a/build-tools/path/linux-x86/cmp b/build-tools/path/linux-x86/cmp new file mode 120000 index 0000000000..70a8836545 --- /dev/null +++ b/build-tools/path/linux-x86/cmp @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/cmp \ No newline at end of file diff --git a/build-tools/path/linux-x86/comm b/build-tools/path/linux-x86/comm new file mode 120000 index 0000000000..23cf94e5b8 --- /dev/null +++ b/build-tools/path/linux-x86/comm @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/comm \ No newline at end of file diff --git a/build-tools/path/linux-x86/cp b/build-tools/path/linux-x86/cp new file mode 120000 index 0000000000..067aac839c --- /dev/null +++ b/build-tools/path/linux-x86/cp @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/cp \ No newline at end of file diff --git a/build-tools/path/linux-x86/cpio b/build-tools/path/linux-x86/cpio new file mode 120000 index 0000000000..4bf115b82b --- /dev/null +++ b/build-tools/path/linux-x86/cpio @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/linux-x86/bin/toybox \ No newline at end of file diff --git a/build-tools/path/linux-x86/cut b/build-tools/path/linux-x86/cut new file mode 120000 index 0000000000..e7de104453 --- /dev/null +++ b/build-tools/path/linux-x86/cut @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/cut \ No newline at end of file diff --git a/build-tools/path/linux-x86/date b/build-tools/path/linux-x86/date new file mode 120000 index 0000000000..b45dc5c33f --- /dev/null +++ b/build-tools/path/linux-x86/date @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/date \ No newline at end of file diff --git a/build-tools/path/linux-x86/depmod b/build-tools/path/linux-x86/depmod new file mode 120000 index 0000000000..52e5c9e637 --- /dev/null +++ b/build-tools/path/linux-x86/depmod @@ -0,0 +1 @@ +../../../../prebuilts/kernel-build-tools/linux-x86/bin/depmod \ No newline at end of file diff --git a/build-tools/path/linux-x86/diff b/build-tools/path/linux-x86/diff new file mode 120000 index 0000000000..4bf115b82b --- /dev/null +++ b/build-tools/path/linux-x86/diff @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/linux-x86/bin/toybox \ No newline at end of file diff --git a/build-tools/path/linux-x86/dirname b/build-tools/path/linux-x86/dirname new file mode 120000 index 0000000000..a021dfcb98 --- /dev/null +++ b/build-tools/path/linux-x86/dirname @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/dirname \ No newline at end of file diff --git a/build-tools/path/linux-x86/dtc b/build-tools/path/linux-x86/dtc new file mode 120000 index 0000000000..b88ace0dbc --- /dev/null +++ b/build-tools/path/linux-x86/dtc @@ -0,0 +1 @@ +../../../../prebuilts/kernel-build-tools/linux-x86/bin/dtc \ No newline at end of file diff --git a/build-tools/path/linux-x86/du b/build-tools/path/linux-x86/du new file mode 120000 index 0000000000..7950478c78 --- /dev/null +++ b/build-tools/path/linux-x86/du @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/du \ No newline at end of file diff --git a/build-tools/path/linux-x86/e2fsck b/build-tools/path/linux-x86/e2fsck new file mode 120000 index 0000000000..60a9456442 --- /dev/null +++ b/build-tools/path/linux-x86/e2fsck @@ -0,0 +1 @@ +../../../../prebuilts/kernel-build-tools/linux-x86/bin/e2fsck \ No newline at end of file diff --git a/build-tools/path/linux-x86/e2fsdroid b/build-tools/path/linux-x86/e2fsdroid new file mode 120000 index 0000000000..00ba7e189a --- /dev/null +++ b/build-tools/path/linux-x86/e2fsdroid @@ -0,0 +1 @@ +../../../../prebuilts/kernel-build-tools/linux-x86/bin/e2fsdroid \ No newline at end of file diff --git a/build-tools/path/linux-x86/echo b/build-tools/path/linux-x86/echo new file mode 120000 index 0000000000..06a21e874a --- /dev/null +++ b/build-tools/path/linux-x86/echo @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/echo \ No newline at end of file diff --git a/build-tools/path/linux-x86/egrep b/build-tools/path/linux-x86/egrep new file mode 120000 index 0000000000..5eda452d69 --- /dev/null +++ b/build-tools/path/linux-x86/egrep @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/egrep \ No newline at end of file diff --git a/build-tools/path/linux-x86/env b/build-tools/path/linux-x86/env new file mode 120000 index 0000000000..6c6cc0bfdf --- /dev/null +++ b/build-tools/path/linux-x86/env @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/env \ No newline at end of file diff --git a/build-tools/path/linux-x86/expr b/build-tools/path/linux-x86/expr new file mode 120000 index 0000000000..4bf115b82b --- /dev/null +++ b/build-tools/path/linux-x86/expr @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/linux-x86/bin/toybox \ No newline at end of file diff --git a/build-tools/path/linux-x86/find b/build-tools/path/linux-x86/find new file mode 120000 index 0000000000..e8202b240c --- /dev/null +++ b/build-tools/path/linux-x86/find @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/find \ No newline at end of file diff --git a/build-tools/path/linux-x86/flex b/build-tools/path/linux-x86/flex new file mode 120000 index 0000000000..7c4992ce7a --- /dev/null +++ b/build-tools/path/linux-x86/flex @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/linux-x86/bin/flex \ No newline at end of file diff --git a/build-tools/path/linux-x86/getconf b/build-tools/path/linux-x86/getconf new file mode 120000 index 0000000000..4b78d9babd --- /dev/null +++ b/build-tools/path/linux-x86/getconf @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/getconf \ No newline at end of file diff --git a/build-tools/path/linux-x86/grep b/build-tools/path/linux-x86/grep new file mode 120000 index 0000000000..8e6775e18a --- /dev/null +++ b/build-tools/path/linux-x86/grep @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/grep \ No newline at end of file diff --git a/build-tools/path/linux-x86/gzip b/build-tools/path/linux-x86/gzip new file mode 120000 index 0000000000..a233f9c41c --- /dev/null +++ b/build-tools/path/linux-x86/gzip @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/gzip \ No newline at end of file diff --git a/build-tools/path/linux-x86/head b/build-tools/path/linux-x86/head new file mode 120000 index 0000000000..3d0375c299 --- /dev/null +++ b/build-tools/path/linux-x86/head @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/head \ No newline at end of file diff --git a/build-tools/path/linux-x86/hostname b/build-tools/path/linux-x86/hostname new file mode 120000 index 0000000000..86c2c44c5d --- /dev/null +++ b/build-tools/path/linux-x86/hostname @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/hostname \ No newline at end of file diff --git a/build-tools/path/linux-x86/id b/build-tools/path/linux-x86/id new file mode 120000 index 0000000000..66a0d67cde --- /dev/null +++ b/build-tools/path/linux-x86/id @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/id \ No newline at end of file diff --git a/build-tools/path/linux-x86/img2simg b/build-tools/path/linux-x86/img2simg new file mode 120000 index 0000000000..b0abe45d0c --- /dev/null +++ b/build-tools/path/linux-x86/img2simg @@ -0,0 +1 @@ +../../../../prebuilts/kernel-build-tools/linux-x86/bin/img2simg \ No newline at end of file diff --git a/build-tools/path/linux-x86/ln b/build-tools/path/linux-x86/ln new file mode 120000 index 0000000000..0d882909fa --- /dev/null +++ b/build-tools/path/linux-x86/ln @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/ln \ No newline at end of file diff --git a/build-tools/path/linux-x86/lpmake b/build-tools/path/linux-x86/lpmake new file mode 120000 index 0000000000..02f9a50432 --- /dev/null +++ b/build-tools/path/linux-x86/lpmake @@ -0,0 +1 @@ +../../../../prebuilts/kernel-build-tools/linux-x86/bin/lpmake \ No newline at end of file diff --git a/build-tools/path/linux-x86/ls b/build-tools/path/linux-x86/ls new file mode 120000 index 0000000000..b5f162e873 --- /dev/null +++ b/build-tools/path/linux-x86/ls @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/ls \ No newline at end of file diff --git a/build-tools/path/linux-x86/lz4 b/build-tools/path/linux-x86/lz4 new file mode 120000 index 0000000000..d2502e205c --- /dev/null +++ b/build-tools/path/linux-x86/lz4 @@ -0,0 +1 @@ +../../../../prebuilts/kernel-build-tools/linux-x86/bin/lz4 \ No newline at end of file diff --git a/build-tools/path/linux-x86/m4 b/build-tools/path/linux-x86/m4 new file mode 120000 index 0000000000..5944dbf4a4 --- /dev/null +++ b/build-tools/path/linux-x86/m4 @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/linux-x86/bin/m4 \ No newline at end of file diff --git a/build-tools/path/linux-x86/make b/build-tools/path/linux-x86/make new file mode 120000 index 0000000000..0c587ac19b --- /dev/null +++ b/build-tools/path/linux-x86/make @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/linux-x86/bin/make \ No newline at end of file diff --git a/build-tools/path/linux-x86/md5sum b/build-tools/path/linux-x86/md5sum new file mode 120000 index 0000000000..bc1bd4a509 --- /dev/null +++ b/build-tools/path/linux-x86/md5sum @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/md5sum \ No newline at end of file diff --git a/build-tools/path/linux-x86/mkbootfs b/build-tools/path/linux-x86/mkbootfs new file mode 120000 index 0000000000..15e0176592 --- /dev/null +++ b/build-tools/path/linux-x86/mkbootfs @@ -0,0 +1 @@ +../../../../prebuilts/kernel-build-tools/linux-x86/bin/mkbootfs \ No newline at end of file diff --git a/build-tools/path/linux-x86/mkdir b/build-tools/path/linux-x86/mkdir new file mode 120000 index 0000000000..acc136705a --- /dev/null +++ b/build-tools/path/linux-x86/mkdir @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/mkdir \ No newline at end of file diff --git a/build-tools/path/linux-x86/mkdtboimg.py b/build-tools/path/linux-x86/mkdtboimg.py new file mode 120000 index 0000000000..439892cb39 --- /dev/null +++ b/build-tools/path/linux-x86/mkdtboimg.py @@ -0,0 +1 @@ +../../../../prebuilts/kernel-build-tools/linux-x86/bin/mkdtboimg.py \ No newline at end of file diff --git a/build-tools/path/linux-x86/mke2fs b/build-tools/path/linux-x86/mke2fs new file mode 120000 index 0000000000..f606429f47 --- /dev/null +++ b/build-tools/path/linux-x86/mke2fs @@ -0,0 +1 @@ +../../../../prebuilts/kernel-build-tools/linux-x86/bin/mke2fs \ No newline at end of file diff --git a/build-tools/path/linux-x86/mktemp b/build-tools/path/linux-x86/mktemp new file mode 120000 index 0000000000..81f6174f90 --- /dev/null +++ b/build-tools/path/linux-x86/mktemp @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/mktemp \ No newline at end of file diff --git a/build-tools/path/linux-x86/mkuserimg_mke2fs b/build-tools/path/linux-x86/mkuserimg_mke2fs new file mode 120000 index 0000000000..9d1aa13dc2 --- /dev/null +++ b/build-tools/path/linux-x86/mkuserimg_mke2fs @@ -0,0 +1 @@ +../../../../prebuilts/kernel-build-tools/linux-x86/bin/mkuserimg_mke2fs \ No newline at end of file diff --git a/build-tools/path/linux-x86/mv b/build-tools/path/linux-x86/mv new file mode 120000 index 0000000000..975f1a1679 --- /dev/null +++ b/build-tools/path/linux-x86/mv @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/mv \ No newline at end of file diff --git a/build-tools/path/linux-x86/nproc b/build-tools/path/linux-x86/nproc new file mode 120000 index 0000000000..4bf115b82b --- /dev/null +++ b/build-tools/path/linux-x86/nproc @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/linux-x86/bin/toybox \ No newline at end of file diff --git a/build-tools/path/linux-x86/od b/build-tools/path/linux-x86/od new file mode 120000 index 0000000000..ac0990376d --- /dev/null +++ b/build-tools/path/linux-x86/od @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/od \ No newline at end of file diff --git a/build-tools/path/linux-x86/openssl b/build-tools/path/linux-x86/openssl new file mode 120000 index 0000000000..19d6156729 --- /dev/null +++ b/build-tools/path/linux-x86/openssl @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/linux-x86/bin/openssl \ No newline at end of file diff --git a/build-tools/path/linux-x86/pahole b/build-tools/path/linux-x86/pahole new file mode 120000 index 0000000000..bfdd7e17df --- /dev/null +++ b/build-tools/path/linux-x86/pahole @@ -0,0 +1 @@ +../../../../prebuilts/kernel-build-tools/linux-x86/bin/pahole \ No newline at end of file diff --git a/build-tools/path/linux-x86/paste b/build-tools/path/linux-x86/paste new file mode 120000 index 0000000000..74d0f2b73d --- /dev/null +++ b/build-tools/path/linux-x86/paste @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/paste \ No newline at end of file diff --git a/build-tools/path/linux-x86/pgrep b/build-tools/path/linux-x86/pgrep new file mode 120000 index 0000000000..684fadf010 --- /dev/null +++ b/build-tools/path/linux-x86/pgrep @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/pgrep \ No newline at end of file diff --git a/build-tools/path/linux-x86/pkill b/build-tools/path/linux-x86/pkill new file mode 120000 index 0000000000..5ff766ac6f --- /dev/null +++ b/build-tools/path/linux-x86/pkill @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/pkill \ No newline at end of file diff --git a/build-tools/path/linux-x86/ps b/build-tools/path/linux-x86/ps new file mode 120000 index 0000000000..fb750e9355 --- /dev/null +++ b/build-tools/path/linux-x86/ps @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/ps \ No newline at end of file diff --git a/build-tools/path/linux-x86/pwd b/build-tools/path/linux-x86/pwd new file mode 120000 index 0000000000..94938bef02 --- /dev/null +++ b/build-tools/path/linux-x86/pwd @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/pwd \ No newline at end of file diff --git a/build-tools/path/linux-x86/python b/build-tools/path/linux-x86/python new file mode 120000 index 0000000000..8ee52bf7b5 --- /dev/null +++ b/build-tools/path/linux-x86/python @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/python \ No newline at end of file diff --git a/build-tools/path/linux-x86/python2 b/build-tools/path/linux-x86/python2 new file mode 120000 index 0000000000..6dc382c967 --- /dev/null +++ b/build-tools/path/linux-x86/python2 @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/python2 \ No newline at end of file diff --git a/build-tools/path/linux-x86/python2.7 b/build-tools/path/linux-x86/python2.7 new file mode 120000 index 0000000000..a10777833b --- /dev/null +++ b/build-tools/path/linux-x86/python2.7 @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/python2.7 \ No newline at end of file diff --git a/build-tools/path/linux-x86/python3 b/build-tools/path/linux-x86/python3 new file mode 120000 index 0000000000..e74d9bc61a --- /dev/null +++ b/build-tools/path/linux-x86/python3 @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/python3 \ No newline at end of file diff --git a/build-tools/path/linux-x86/readlink b/build-tools/path/linux-x86/readlink new file mode 120000 index 0000000000..bb56a21571 --- /dev/null +++ b/build-tools/path/linux-x86/readlink @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/readlink \ No newline at end of file diff --git a/build-tools/path/linux-x86/realpath b/build-tools/path/linux-x86/realpath new file mode 120000 index 0000000000..4bf115b82b --- /dev/null +++ b/build-tools/path/linux-x86/realpath @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/linux-x86/bin/toybox \ No newline at end of file diff --git a/build-tools/path/linux-x86/rm b/build-tools/path/linux-x86/rm new file mode 120000 index 0000000000..ed8748addc --- /dev/null +++ b/build-tools/path/linux-x86/rm @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/rm \ No newline at end of file diff --git a/build-tools/path/linux-x86/rmdir b/build-tools/path/linux-x86/rmdir new file mode 120000 index 0000000000..ebcf98366f --- /dev/null +++ b/build-tools/path/linux-x86/rmdir @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/rmdir \ No newline at end of file diff --git a/build-tools/path/linux-x86/sed b/build-tools/path/linux-x86/sed new file mode 120000 index 0000000000..ada77e3282 --- /dev/null +++ b/build-tools/path/linux-x86/sed @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/sed \ No newline at end of file diff --git a/build-tools/path/linux-x86/seq b/build-tools/path/linux-x86/seq new file mode 120000 index 0000000000..38cc8850a4 --- /dev/null +++ b/build-tools/path/linux-x86/seq @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/seq \ No newline at end of file diff --git a/build-tools/path/linux-x86/setsid b/build-tools/path/linux-x86/setsid new file mode 120000 index 0000000000..c2a339d9b0 --- /dev/null +++ b/build-tools/path/linux-x86/setsid @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/setsid \ No newline at end of file diff --git a/build-tools/path/linux-x86/sha1sum b/build-tools/path/linux-x86/sha1sum new file mode 120000 index 0000000000..be9f137e6d --- /dev/null +++ b/build-tools/path/linux-x86/sha1sum @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/sha1sum \ No newline at end of file diff --git a/build-tools/path/linux-x86/sha256sum b/build-tools/path/linux-x86/sha256sum new file mode 120000 index 0000000000..98bfbad225 --- /dev/null +++ b/build-tools/path/linux-x86/sha256sum @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/sha256sum \ No newline at end of file diff --git a/build-tools/path/linux-x86/sha512sum b/build-tools/path/linux-x86/sha512sum new file mode 120000 index 0000000000..d7f03830bf --- /dev/null +++ b/build-tools/path/linux-x86/sha512sum @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/sha512sum \ No newline at end of file diff --git a/build-tools/path/linux-x86/simg2img b/build-tools/path/linux-x86/simg2img new file mode 120000 index 0000000000..e22520d432 --- /dev/null +++ b/build-tools/path/linux-x86/simg2img @@ -0,0 +1 @@ +../../../../prebuilts/kernel-build-tools/linux-x86/bin/simg2img \ No newline at end of file diff --git a/build-tools/path/linux-x86/sleep b/build-tools/path/linux-x86/sleep new file mode 120000 index 0000000000..85ec0c8e0f --- /dev/null +++ b/build-tools/path/linux-x86/sleep @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/sleep \ No newline at end of file diff --git a/build-tools/path/linux-x86/sort b/build-tools/path/linux-x86/sort new file mode 120000 index 0000000000..bc9e63959c --- /dev/null +++ b/build-tools/path/linux-x86/sort @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/sort \ No newline at end of file diff --git a/build-tools/path/linux-x86/stat b/build-tools/path/linux-x86/stat new file mode 120000 index 0000000000..f535efccc0 --- /dev/null +++ b/build-tools/path/linux-x86/stat @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/stat \ No newline at end of file diff --git a/build-tools/path/linux-x86/swig b/build-tools/path/linux-x86/swig new file mode 120000 index 0000000000..bda36aa59e --- /dev/null +++ b/build-tools/path/linux-x86/swig @@ -0,0 +1 @@ +../../../../prebuilts/kernel-build-tools/linux-x86/bin/swig \ No newline at end of file diff --git a/build-tools/path/linux-x86/tail b/build-tools/path/linux-x86/tail new file mode 120000 index 0000000000..12c0d08f64 --- /dev/null +++ b/build-tools/path/linux-x86/tail @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/tail \ No newline at end of file diff --git a/build-tools/path/linux-x86/tee b/build-tools/path/linux-x86/tee new file mode 120000 index 0000000000..feda8f56fc --- /dev/null +++ b/build-tools/path/linux-x86/tee @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/tee \ No newline at end of file diff --git a/build-tools/path/linux-x86/test b/build-tools/path/linux-x86/test new file mode 120000 index 0000000000..7f1cd3d1c5 --- /dev/null +++ b/build-tools/path/linux-x86/test @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/test \ No newline at end of file diff --git a/build-tools/path/linux-x86/timeout b/build-tools/path/linux-x86/timeout new file mode 120000 index 0000000000..1fa683dafa --- /dev/null +++ b/build-tools/path/linux-x86/timeout @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/timeout \ No newline at end of file diff --git a/build-tools/path/linux-x86/touch b/build-tools/path/linux-x86/touch new file mode 120000 index 0000000000..73a656cc5c --- /dev/null +++ b/build-tools/path/linux-x86/touch @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/touch \ No newline at end of file diff --git a/build-tools/path/linux-x86/tr b/build-tools/path/linux-x86/tr new file mode 120000 index 0000000000..4bf115b82b --- /dev/null +++ b/build-tools/path/linux-x86/tr @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/linux-x86/bin/toybox \ No newline at end of file diff --git a/build-tools/path/linux-x86/true b/build-tools/path/linux-x86/true new file mode 120000 index 0000000000..1322b7f423 --- /dev/null +++ b/build-tools/path/linux-x86/true @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/true \ No newline at end of file diff --git a/build-tools/path/linux-x86/truncate b/build-tools/path/linux-x86/truncate new file mode 120000 index 0000000000..5372746ea1 --- /dev/null +++ b/build-tools/path/linux-x86/truncate @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/truncate \ No newline at end of file diff --git a/build-tools/path/linux-x86/tune2fs b/build-tools/path/linux-x86/tune2fs new file mode 120000 index 0000000000..c746297435 --- /dev/null +++ b/build-tools/path/linux-x86/tune2fs @@ -0,0 +1 @@ +../../../../prebuilts/kernel-build-tools/linux-x86/bin/tune2fs \ No newline at end of file diff --git a/build-tools/path/linux-x86/ufdt_apply_overlay b/build-tools/path/linux-x86/ufdt_apply_overlay new file mode 120000 index 0000000000..3c018ae03a --- /dev/null +++ b/build-tools/path/linux-x86/ufdt_apply_overlay @@ -0,0 +1 @@ +../../../../prebuilts/kernel-build-tools/linux-x86/bin/ufdt_apply_overlay \ No newline at end of file diff --git a/build-tools/path/linux-x86/uname b/build-tools/path/linux-x86/uname new file mode 120000 index 0000000000..15a05231ef --- /dev/null +++ b/build-tools/path/linux-x86/uname @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/uname \ No newline at end of file diff --git a/build-tools/path/linux-x86/uniq b/build-tools/path/linux-x86/uniq new file mode 120000 index 0000000000..f1a3fe90a9 --- /dev/null +++ b/build-tools/path/linux-x86/uniq @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/uniq \ No newline at end of file diff --git a/build-tools/path/linux-x86/unix2dos b/build-tools/path/linux-x86/unix2dos new file mode 120000 index 0000000000..09ca53628c --- /dev/null +++ b/build-tools/path/linux-x86/unix2dos @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/unix2dos \ No newline at end of file diff --git a/build-tools/path/linux-x86/wc b/build-tools/path/linux-x86/wc new file mode 120000 index 0000000000..d6de485b75 --- /dev/null +++ b/build-tools/path/linux-x86/wc @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/wc \ No newline at end of file diff --git a/build-tools/path/linux-x86/which b/build-tools/path/linux-x86/which new file mode 120000 index 0000000000..c560b07328 --- /dev/null +++ b/build-tools/path/linux-x86/which @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/which \ No newline at end of file diff --git a/build-tools/path/linux-x86/whoami b/build-tools/path/linux-x86/whoami new file mode 120000 index 0000000000..d087e09a18 --- /dev/null +++ b/build-tools/path/linux-x86/whoami @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/whoami \ No newline at end of file diff --git a/build-tools/path/linux-x86/xargs b/build-tools/path/linux-x86/xargs new file mode 120000 index 0000000000..cc37ad7b36 --- /dev/null +++ b/build-tools/path/linux-x86/xargs @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/xargs \ No newline at end of file diff --git a/build-tools/path/linux-x86/xxd b/build-tools/path/linux-x86/xxd new file mode 120000 index 0000000000..9bd64f2745 --- /dev/null +++ b/build-tools/path/linux-x86/xxd @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/xxd \ No newline at end of file diff --git a/build-tools/path/linux-x86/xz b/build-tools/path/linux-x86/xz new file mode 120000 index 0000000000..5de3fa13ab --- /dev/null +++ b/build-tools/path/linux-x86/xz @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/xz \ No newline at end of file diff --git a/build-tools/path/linux-x86/xzcat b/build-tools/path/linux-x86/xzcat new file mode 120000 index 0000000000..6ef31644f4 --- /dev/null +++ b/build-tools/path/linux-x86/xzcat @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/xzcat \ No newline at end of file diff --git a/build-tools/path/linux-x86/zipinfo b/build-tools/path/linux-x86/zipinfo new file mode 120000 index 0000000000..b3dd7642e0 --- /dev/null +++ b/build-tools/path/linux-x86/zipinfo @@ -0,0 +1 @@ +../../../../prebuilts/build-tools/path/linux-x86/zipinfo \ No newline at end of file diff --git a/build-tools/sysroot b/build-tools/sysroot new file mode 120000 index 0000000000..d0298f10fe --- /dev/null +++ b/build-tools/sysroot @@ -0,0 +1 @@ +../../prebuilts/gcc/linux-x86/host/x86_64-linux-glibc2.17-4.8/sysroot \ No newline at end of file diff --git a/build.config b/build.config new file mode 120000 index 0000000000..7c5f6379b3 --- /dev/null +++ b/build.config @@ -0,0 +1 @@ +../build.config \ No newline at end of file diff --git a/build.config.net_test b/build.config.net_test new file mode 100644 index 0000000000..d6527a4430 --- /dev/null +++ b/build.config.net_test @@ -0,0 +1,12 @@ +ARCH=um +SUBARCH=x86_64 +CC=${CC:-clang} +CLANG_TRIPLE=x86_64-linux-gnu- +CROSS_COMPILE=x86_64-linux-androidkernel- +EXTRA_CMDS='' +KERNEL_DIR=$(. $(dirname $0)/build.config && echo $KERNEL_DIR || echo common) +CLANG_PREBUILT_BIN=prebuilts-master/clang/host/linux-x86/clang-r383902/bin +LINUX_GCC_CROSS_COMPILE_PREBUILTS_BIN=prebuilts/gcc/linux-x86/x86/x86_64-linux-android-4.9/bin +FILES=" +linux +" diff --git a/build.sh b/build.sh new file mode 100755 index 0000000000..2be333fc7b --- /dev/null +++ b/build.sh @@ -0,0 +1,1312 @@ +#!/bin/bash + +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Usage: +# build/build.sh * +# or: +# To define custom out and dist directories: +# OUT_DIR= DIST_DIR= build/build.sh * +# To use a custom build config: +# BUILD_CONFIG= * +# +# Examples: +# To define custom out and dist directories: +# OUT_DIR=output DIST_DIR=dist build/build.sh -j24 V=1 +# To use a custom build config: +# BUILD_CONFIG=common/build.config.gki.aarch64 build/build.sh -j24 V=1 +# +# The following environment variables are considered during execution: +# +# BUILD_CONFIG +# Build config file to initialize the build environment from. The location +# is to be defined relative to the repo root directory. +# Defaults to 'build.config'. +# +# BUILD_CONFIG_FRAGMENTS +# A whitespace-separated list of additional build config fragments to be +# sourced after the main build config file. Typically used for sanitizers or +# other special builds. +# +# OUT_DIR +# Base output directory for the kernel build. +# Defaults to /out/. +# +# DIST_DIR +# Base output directory for the kernel distribution. +# Defaults to /dist +# +# MAKE_GOALS +# List of targets passed to Make when compiling the kernel. +# Typically: Image, modules, and a DTB (if applicable). +# +# EXT_MODULES +# Space separated list of external kernel modules to be build. +# +# UNSTRIPPED_MODULES +# Space separated list of modules to be copied to /unstripped +# for debugging purposes. +# +# COMPRESS_UNSTRIPPED_MODULES +# If set to "1", then compress the unstripped modules into a tarball. +# +# CC +# Override compiler to be used. (e.g. CC=clang) Specifying CC=gcc +# effectively unsets CC to fall back to the default gcc detected by kbuild +# (including any target triplet). To use a custom 'gcc' from PATH, use an +# absolute path, e.g. CC=/usr/local/bin/gcc +# +# LD +# Override linker (flags) to be used. +# +# HERMETIC_TOOLCHAIN +# When set, the PATH during kernel build will be restricted to a set of +# known prebuilt directories and selected host tools that are usually not +# provided by prebuilt toolchains. +# +# ADDITIONAL_HOST_TOOLS +# A whitespace separated set of tools that will be allowed to be used from +# the host when running the build with HERMETIC_TOOLCHAIN=1. +# +# ABI_DEFINITION +# Location of the abi definition file relative to /KERNEL_DIR +# If defined (usually in build.config), also copy that abi definition to +# /dist/abi.xml when creating the distribution. +# +# KMI_SYMBOL_LIST +# Location of the main KMI symbol list file relative to +# /KERNEL_DIR If defined (usually in build.config), also copy +# that symbol list definition to /dist/abi_symbollist when +# creating the distribution. +# +# ADDITIONAL_KMI_SYMBOL_LISTS +# Location of secondary KMI symbol list files relative to +# /KERNEL_DIR. If defined, these additional symbol lists will be +# appended to the main one before proceeding to the distribution creation. +# +# KMI_ENFORCED +# This is an indicative option to signal that KMI is enforced in this build +# config. If set to "1", downstream KMI checking tools might respect it and +# react to it by failing if KMI differences are detected. +# +# GENERATE_VMLINUX_BTF +# If set to "1", generate a vmlinux.btf that is stripped off any debug +# symbols, but contains type and symbol information within a .BTF section. +# This is suitable for ABI analysis through BTF. +# +# Environment variables to influence the stages of the kernel build. +# +# SKIP_MRPROPER +# if set to "1", skip `make mrproper` +# +# SKIP_DEFCONFIG +# if set to "1", skip `make defconfig` +# +# SKIP_IF_VERSION_MATCHES +# if defined, skip compiling anything if the kernel version in vmlinux +# matches the expected kernel version. This is useful for mixed build, where +# GKI kernel does not change frequently and we can simply skip everything +# in build.sh. Note: if the expected version string contains "dirty", then +# this flag would have not cause build.sh to exit early. +# +# PRE_DEFCONFIG_CMDS +# Command evaluated before `make defconfig` +# +# POST_DEFCONFIG_CMDS +# Command evaluated after `make defconfig` and before `make`. +# +# POST_KERNEL_BUILD_CMDS +# Command evaluated after `make`. +# +# LTO=[full|thin|none] +# If set to "full", force any kernel with LTO_CLANG support to be built +# with full LTO, which is the most optimized method. This is the default, +# but can result in very slow build times, especially when building +# incrementally. (This mode does not require CFI to be disabled.) +# If set to "thin", force any kernel with LTO_CLANG support to be built +# with ThinLTO, which trades off some optimizations for incremental build +# speed. This is nearly always what you want for local development. (This +# mode does not require CFI to be disabled.) +# If set to "none", force any kernel with LTO_CLANG support to be built +# without any LTO (upstream default), which results in no optimizations +# and also disables LTO-dependent features like CFI. This mode is not +# recommended because CFI will not be able to catch bugs if it is +# disabled. +# +# TAGS_CONFIG +# if defined, calls ./scripts/tags.sh utility with TAGS_CONFIG as argument +# and exit once tags have been generated +# +# IN_KERNEL_MODULES +# if defined, install kernel modules +# +# SKIP_EXT_MODULES +# if defined, skip building and installing of external modules +# +# DO_NOT_STRIP_MODULES +# if set to "1", keep debug information for distributed modules. +# Note, modules will still be stripped when copied into the ramdisk. +# +# EXTRA_CMDS +# Command evaluated after building and installing kernel and modules. +# +# DIST_CMDS +# Command evaluated after copying files to DIST_DIR +# +# SKIP_CP_KERNEL_HDR +# if defined, skip installing kernel headers. +# +# BUILD_BOOT_IMG +# if defined, build a boot.img binary that can be flashed into the 'boot' +# partition of an Android device. The boot image contains a header as per the +# format defined by https://source.android.com/devices/bootloader/boot-image-header +# followed by several components like kernel, ramdisk, DTB etc. The ramdisk +# component comprises of a GKI ramdisk cpio archive concatenated with a +# vendor ramdisk cpio archive which is then gzipped. It is expected that +# all components are present in ${DIST_DIR}. +# +# When the BUILD_BOOT_IMG flag is defined, the following flags that point to the +# various components needed to build a boot.img also need to be defined. +# - MKBOOTIMG_PATH= +# (defaults to tools/mkbootimg/mkbootimg.py) +# - GKI_RAMDISK_PREBUILT_BINARY= +# - VENDOR_RAMDISK_BINARY= +# - KERNEL_BINARY= +# - BOOT_IMAGE_HEADER_VERSION= +# (defaults to 3) +# - KERNEL_CMDLINE= +# - KERNEL_VENDOR_CMDLINE== 3; boot otherwise> +# - VENDOR_FSTAB= +# - TAGS_OFFSET= +# - RAMDISK_OFFSET= +# If the BOOT_IMAGE_HEADER_VERSION is less than 3, two additional variables must +# be defined: +# - BASE_ADDRESS= +# - PAGE_SIZE= +# If BOOT_IMAGE_HEADER_VERSION >= 3, a vendor_boot image will be built +# unless SKIP_VENDOR_BOOT is defined. A vendor_boot will also be generated if +# BUILD_VENDOR_BOOT_IMG is set. +# +# BUILD_VENDOR_BOOT_IMG is incompatible with SKIP_VENDOR_BOOT, and is effectively a +# nop if BUILD_BOOT_IMG is set. +# - MODULES_LIST= list of modules to use for +# vendor_boot.modules.load. If this property is not set, then the default +# modules.load is used. +# - TRIM_UNUSED_MODULES. If set, then modules not mentioned in +# modules.load are removed from initramfs. If MODULES_LIST is unset, then +# having this variable set effectively becomes a no-op. +# - MODULES_BLOCKLIST= A list of modules which are +# blocked from being loaded. This file is copied directly to staging directory, +# and should be in the format: +# blocklist module_name +# - MKBOOTIMG_EXTRA_ARGS= +# Refer to: ./mkbootimg.py --help +# If BOOT_IMAGE_HEADER_VERSION >= 4, the following variable can be defined: +# - VENDOR_BOOTCONFIG= +# - INITRAMFS_VENDOR_RAMDISK_FRAGMENT_NAME= +# If BUILD_INITRAMFS is specified, then build the .ko and depmod files as +# a standalone vendor ramdisk fragment named as the given string. +# - INITRAMFS_VENDOR_RAMDISK_FRAGMENT_MKBOOTIMG_ARGS= +# Refer to: https://source.android.com/devices/bootloader/partitions/vendor-boot-partitions#mkbootimg-arguments +# +# VENDOR_RAMDISK_CMDS +# When building vendor boot image, VENDOR_RAMDISK_CMDS enables the build +# config file to specify command(s) for further altering the prebuilt vendor +# ramdisk binary. For example, the build config file could add firmware files +# on the vendor ramdisk (lib/firmware) for testing purposes. +# +# AVB_SIGN_BOOT_IMG +# if defined, sign the boot image using the AVB_BOOT_KEY. Refer to +# https://android.googlesource.com/platform/external/avb/+/master/README.md +# for details on what Android Verified Boot is and how it works. The kernel +# prebuilt tool `avbtool` is used for signing. +# +# When AVB_SIGN_BOOT_IMG is defined, the following flags need to be +# defined: +# - AVB_BOOT_PARTITION_SIZE= +# - AVB_BOOT_KEY= The Android test +# key has been uploaded to the kernel/prebuilts/build-tools project here: +# https://android.googlesource.com/kernel/prebuilts/build-tools/+/refs/heads/master/linux-x86/share/avb +# - AVB_BOOT_ALGORITHM= e.g. SHA256_RSA2048. For the +# full list of supported algorithms, refer to the enum AvbAlgorithmType in +# https://android.googlesource.com/platform/external/avb/+/refs/heads/master/libavb/avb_crypto.h +# +# BUILD_INITRAMFS +# if set to "1", build a ramdisk containing all .ko files and resulting +# depmod artifacts +# +# MODULES_OPTIONS +# A /lib/modules/modules.options file is created on the ramdisk containing +# the contents of this variable, lines should be of the form: options +# = = ... +# +# MODULES_ORDER +# location of an optional file containing the list of modules that are +# expected to be built for the current configuration, in the modules.order +# format, relative to the kernel source tree. +# +# GKI_MODULES_LIST +# location of an optional file containing the list of GKI modules, relative +# to the kernel source tree. This should be set in downstream builds to +# ensure the ABI tooling correctly differentiates vendor/OEM modules and GKI +# modules. This should not be set in the upstream GKI build.config. +# +# VENDOR_DLKM_MODULES_LIST +# location (relative to the repo root directory) of an optional file +# containing the list of kernel modules which shall be copied into a +# vendor_dlkm partition image. Any modules passed into MODULES_LIST which +# become part of the vendor_boot.modules.load will be trimmed from the +# vendor_dlkm.modules.load. +# +# VENDOR_DLKM_MODULES_BLOCKLIST +# location (relative to the repo root directory) of an optional file +# containing a list of modules which are blocked from being loaded. This +# file is copied directly to the staging directory and should be in the +# format: blocklist module_name +# +# VENDOR_DLKM_PROPS +# location (relative to the repo root directory) of a text file containing +# the properties to be used for creation of a vendor_dlkm image +# (filesystem, partition size, etc). If this is not set (and +# VENDOR_DLKM_MODULES_LIST is), a default set of properties will be used +# which assumes an ext4 filesystem and a dynamic partition. +# +# LZ4_RAMDISK +# if set to "1", any ramdisks generated will be lz4 compressed instead of +# gzip compressed. +# +# TRIM_NONLISTED_KMI +# if set to "1", enable the CONFIG_UNUSED_KSYMS_WHITELIST kernel config +# option to un-export from the build any un-used and non-symbol-listed +# (as per KMI_SYMBOL_LIST) symbol. +# +# KMI_SYMBOL_LIST_STRICT_MODE +# if set to "1", add a build-time check between the KMI_SYMBOL_LIST and the +# KMI resulting from the build, to ensure they match 1-1. +# +# KMI_STRICT_MODE_OBJECTS +# optional list of objects to consider for the KMI_SYMBOL_LIST_STRICT_MODE +# check. Defaults to 'vmlinux'. +# +# GKI_DIST_DIR +# optional directory from which to copy GKI artifacts into DIST_DIR +# +# GKI_BUILD_CONFIG +# If set, builds a second set of kernel images using GKI_BUILD_CONFIG to +# perform a "mixed build." Mixed builds creates "GKI kernel" and "vendor +# modules" from two different trees. The GKI kernel tree can be the Android +# Common Kernel and the vendor modules tree can be a complete vendor kernel +# tree. GKI_DIST_DIR (above) is set and the GKI kernel's DIST output is +# copied to this DIST output. This allows a vendor tree kernel image to be +# effectively discarded and a GKI kernel Image used from an Android Common +# Kernel. Any variables prefixed with GKI_ are passed into into the GKI +# kernel's build.sh invocation. +# +# This is incompatible with GKI_PREBUILTS_DIR. +# +# GKI_PREBUILTS_DIR +# If set, copies an existing set of GKI kernel binaries to the DIST_DIR to +# perform a "mixed build," as with GKI_BUILD_CONFIG. This allows you to +# skip the additional compilation, if interested. +# +# This is incompatible with GKI_BUILD_CONFIG. +# +# The following must be present: +# vmlinux +# System.map +# vmlinux.symvers +# modules.builtin +# modules.builtin.modinfo +# Image.lz4 +# +# Note: For historic reasons, internally, OUT_DIR will be copied into +# COMMON_OUT_DIR, and OUT_DIR will be then set to +# ${COMMON_OUT_DIR}/${KERNEL_DIR}. This has been done to accommodate existing +# build.config files that expect ${OUT_DIR} to point to the output directory of +# the kernel build. +# +# The kernel is built in ${COMMON_OUT_DIR}/${KERNEL_DIR}. +# Out-of-tree modules are built in ${COMMON_OUT_DIR}/${EXT_MOD} where +# ${EXT_MOD} is the path to the module source code. + +set -e + +# Save environment for mixed build support. +OLD_ENVIRONMENT=$(mktemp) +export -p > ${OLD_ENVIRONMENT} + +# rel_path +# Generate relative directory path to reach directory from +function rel_path() { + local to=$1 + local from=$2 + local path= + local stem= + local prevstem= + [ -n "$to" ] || return 1 + [ -n "$from" ] || return 1 + to=$(readlink -e "$to") + from=$(readlink -e "$from") + [ -n "$to" ] || return 1 + [ -n "$from" ] || return 1 + stem=${from}/ + while [ "${to#$stem}" == "${to}" -a "${stem}" != "${prevstem}" ]; do + prevstem=$stem + stem=$(readlink -e "${stem}/..") + [ "${stem%/}" == "${stem}" ] && stem=${stem}/ + path=${path}../ + done + echo ${path}${to#$stem} +} + +# $1 directory of kernel modules ($1/lib/modules/x.y) +# $2 flags to pass to depmod +# $3 kernel version +function run_depmod() { + ( + local ramdisk_dir=$1 + local DEPMOD_OUTPUT + + cd ${ramdisk_dir} + if ! DEPMOD_OUTPUT="$(depmod $2 -F ${DIST_DIR}/System.map -b . $3 2>&1)"; then + echo "$DEPMOD_OUTPUT" >&2 + exit 1 + fi + echo "$DEPMOD_OUTPUT" + if { echo "$DEPMOD_OUTPUT" | grep -q "needs unknown symbol"; }; then + echo "ERROR: kernel module(s) need unknown symbol(s)" >&2 + exit 1 + fi + ) +} + +# $1 MODULES_LIST, +# $2 MODULES_STAGING_DIR +# $3 IMAGE_STAGING_DIR +# $4 MODULES_BLOCKLIST, +# $5 flags to pass to depmod +function create_modules_staging() { + local modules_list_file=$1 + local src_dir=$(echo $2/lib/modules/*) + local version=$(basename "${src_dir}") + local dest_dir=$3/lib/modules/${version} + local dest_stage=$3 + local modules_blocklist_file=$4 + local depmod_flags=$5 + + rm -rf ${dest_dir} + mkdir -p ${dest_dir}/kernel + find ${src_dir}/kernel/ -maxdepth 1 -mindepth 1 \ + -exec cp -r {} ${dest_dir}/kernel/ \; + # The other modules.* files will be generated by depmod + cp ${src_dir}/modules.order ${dest_dir}/modules.order + cp ${src_dir}/modules.builtin ${dest_dir}/modules.builtin + + if [ -n "${EXT_MODULES}" ]; then + mkdir -p ${dest_dir}/extra/ + cp -r ${src_dir}/extra/* ${dest_dir}/extra/ + (cd ${dest_dir}/ && \ + find extra -type f -name "*.ko" | sort >> modules.order) + fi + + if [ "${DO_NOT_STRIP_MODULES}" = "1" ]; then + # strip debug symbols off initramfs modules + find ${dest_dir} -type f -name "*.ko" \ + -exec ${OBJCOPY:-${CROSS_COMPILE}objcopy} --strip-debug {} \; + fi + + if [ -n "${modules_list_file}" ]; then + echo "========================================================" + echo " Reducing modules.order to:" + # Need to make sure we can find modules_list_file from the staging dir + if [[ -f "${ROOT_DIR}/${modules_list_file}" ]]; then + modules_list_file="${ROOT_DIR}/${modules_list_file}" + elif [[ "${modules_list_file}" != /* ]]; then + echo "modules list must be an absolute path or relative to ${ROOT_DIR}: ${modules_list_file}" + exit 1 + elif [[ ! -f "${modules_list_file}" ]]; then + echo "Failed to find modules list: ${modules_list_file}" + exit 1 + fi + + local modules_list_filter=$(mktemp) + local old_modules_list=$(mktemp) + + # Remove all lines starting with "#" (comments) + # Exclamation point makes interpreter ignore the exit code under set -e + ! grep -v "^\#" ${modules_list_file} > ${modules_list_filter} + + # grep the modules.order for any KOs in the modules list + cp ${dest_dir}/modules.order ${old_modules_list} + ! grep -w -f ${modules_list_filter} ${old_modules_list} > ${dest_dir}/modules.order + rm -f ${modules_list_filter} ${old_modules_list} + cat ${dest_dir}/modules.order | sed -e "s/^/ /" + fi + + if [ -n "${modules_blocklist_file}" ]; then + # Need to make sure we can find modules_blocklist_file from the staging dir + if [[ -f "${ROOT_DIR}/${modules_blocklist_file}" ]]; then + modules_blocklist_file="${ROOT_DIR}/${modules_blocklist_file}" + elif [[ "${modules_blocklist_file}" != /* ]]; then + echo "modules blocklist must be an absolute path or relative to ${ROOT_DIR}: ${modules_blocklist_file}" + exit 1 + elif [[ ! -f "${modules_blocklist_file}" ]]; then + echo "Failed to find modules blocklist: ${modules_blocklist_file}" + exit 1 + fi + + cp ${modules_blocklist_file} ${dest_dir}/modules.blocklist + fi + + if [ -n "${TRIM_UNUSED_MODULES}" ]; then + echo "========================================================" + echo " Trimming unused modules" + local used_blocklist_modules=$(mktemp) + if [ -f ${dest_dir}/modules.blocklist ]; then + # TODO: the modules blocklist could contain module aliases instead of the filename + sed -n -E -e 's/blocklist (.+)/\1/p' ${dest_dir}/modules.blocklist > $used_blocklist_modules + fi + + # Trim modules from tree that aren't mentioned in modules.order + ( + cd ${dest_dir} + find * -type f -name "*.ko" | grep -v -w -f modules.order -f $used_blocklist_modules - | xargs -r rm + ) + rm $used_blocklist_modules + fi + + # Re-run depmod to detect any dependencies between in-kernel and external + # modules. Then, create modules.order based on all the modules compiled. + run_depmod ${dest_stage} "${depmod_flags}" "${version}" + cp ${dest_dir}/modules.order ${dest_dir}/modules.load +} + +function build_vendor_dlkm() { + echo "========================================================" + echo " Creating vendor_dlkm image" + + create_modules_staging "${VENDOR_DLKM_MODULES_LIST}" "${MODULES_STAGING_DIR}" \ + "${VENDOR_DLKM_STAGING_DIR}" "${VENDOR_DLKM_MODULES_BLOCKLIST}" + + local vendor_dlkm_modules_root_dir=$(echo ${VENDOR_DLKM_STAGING_DIR}/lib/modules/*) + local vendor_dlkm_modules_load=${vendor_dlkm_modules_root_dir}/modules.load + + # Modules loaded in vendor_boot should not be loaded in vendor_dlkm. + if [ -f ${DIST_DIR}/vendor_boot.modules.load ]; then + local stripped_modules_load="$(mktemp)" + ! grep -x -v -F -f ${DIST_DIR}/vendor_boot.modules.load \ + ${vendor_dlkm_modules_load} > ${stripped_modules_load} + mv -f ${stripped_modules_load} ${vendor_dlkm_modules_load} + fi + + cp ${vendor_dlkm_modules_load} ${DIST_DIR}/vendor_dlkm.modules.load + local vendor_dlkm_props_file + + if [ -z "${VENDOR_DLKM_PROPS}" ]; then + vendor_dlkm_props_file="$(mktemp)" + echo -e "vendor_dlkm_fs_type=ext4\n" >> ${vendor_dlkm_props_file} + echo -e "use_dynamic_partition_size=true\n" >> ${vendor_dlkm_props_file} + echo -e "ext_mkuserimg=mkuserimg_mke2fs\n" >> ${vendor_dlkm_props_file} + echo -e "ext4_share_dup_blocks=true\n" >> ${vendor_dlkm_props_file} + else + vendor_dlkm_props_file="${VENDOR_DLKM_PROPS}" + if [[ -f "${ROOT_DIR}/${vendor_dlkm_props_file}" ]]; then + vendor_dlkm_props_file="${ROOT_DIR}/${vendor_dlkm_props_file}" + elif [[ "${vendor_dlkm_props_file}" != /* ]]; then + echo "VENDOR_DLKM_PROPS must be an absolute path or relative to ${ROOT_DIR}: ${vendor_dlkm_props_file}" + exit 1 + elif [[ ! -f "${vendor_dlkm_props_file}" ]]; then + echo "Failed to find VENDOR_DLKM_PROPS: ${vendor_dlkm_props_file}" + exit 1 + fi + fi + build_image "${VENDOR_DLKM_STAGING_DIR}" "${vendor_dlkm_props_file}" \ + "${DIST_DIR}/vendor_dlkm.img" /dev/null +} + +export ROOT_DIR=$(readlink -f $(dirname $0)/..) + +# For module file Signing with the kernel (if needed) +FILE_SIGN_BIN=scripts/sign-file +SIGN_SEC=certs/signing_key.pem +SIGN_CERT=certs/signing_key.x509 +SIGN_ALGO=sha512 + +# Save environment parameters before being overwritten by sourcing +# BUILD_CONFIG. +CC_ARG="${CC}" + +source "${ROOT_DIR}/build/_setup_env.sh" + +MAKE_ARGS=( "$@" ) +export MAKEFLAGS="-j$(nproc) ${MAKEFLAGS}" +export MODULES_STAGING_DIR=$(readlink -m ${COMMON_OUT_DIR}/staging) +export MODULES_PRIVATE_DIR=$(readlink -m ${COMMON_OUT_DIR}/private) +export KERNEL_UAPI_HEADERS_DIR=$(readlink -m ${COMMON_OUT_DIR}/kernel_uapi_headers) +export INITRAMFS_STAGING_DIR=${MODULES_STAGING_DIR}/initramfs_staging +export VENDOR_DLKM_STAGING_DIR=${MODULES_STAGING_DIR}/vendor_dlkm_staging +export MKBOOTIMG_STAGING_DIR="${MODULES_STAGING_DIR}/mkbootimg_staging" + +if [ -n "${SKIP_VENDOR_BOOT}" -a -n "${BUILD_VENDOR_BOOT_IMG}" ]; then + echo "ERROR: SKIP_VENDOR_BOOT is incompatible with BUILD_VENDOR_BOOT_IMG." >&2 + exit 1 +fi + +if [ -n "${GKI_BUILD_CONFIG}" ]; then + if [ -n "${GKI_PREBUILTS_DIR}" ]; then + echo "ERROR: GKI_BUILD_CONFIG is incompatible with GKI_PREBUILTS_DIR." >&2 + exit 1 + fi + + GKI_OUT_DIR=${GKI_OUT_DIR:-${COMMON_OUT_DIR}/gki_kernel} + GKI_DIST_DIR=${GKI_DIST_DIR:-${GKI_OUT_DIR}/dist} + + if [[ "${MAKE_GOALS}" =~ image|Image|vmlinux ]]; then + echo " Compiling Image and vmlinux in device kernel is not supported in mixed build mode" + exit 1 + fi + + # Inherit SKIP_MRPROPER, LTO, SKIP_DEFCONFIG unless overridden by corresponding GKI_* variables + GKI_ENVIRON=("SKIP_MRPROPER=${SKIP_MRPROPER}" "LTO=${LTO}" "SKIP_DEFCONFIG=${SKIP_DEFCONFIG}" "SKIP_IF_VERSION_MATCHES=${SKIP_IF_VERSION_MATCHES}") + # Explicitly unset EXT_MODULES since they should be compiled against the device kernel + GKI_ENVIRON+=("EXT_MODULES=") + # Explicitly unset GKI_BUILD_CONFIG in case it was set by in the old environment + # e.g. GKI_BUILD_CONFIG=common/build.config.gki.x86 ./build/build.sh would cause + # gki build recursively + GKI_ENVIRON+=("GKI_BUILD_CONFIG=") + # Any variables prefixed with GKI_ get set without that prefix in the GKI build environment + # e.g. GKI_BUILD_CONFIG=common/build.config.gki.aarch64 -> BUILD_CONFIG=common/build.config.gki.aarch64 + GKI_ENVIRON+=($(export -p | sed -n -E -e 's/.* GKI_([^=]+=.*)$/\1/p' | tr '\n' ' ')) + GKI_ENVIRON+=("OUT_DIR=${GKI_OUT_DIR}") + GKI_ENVIRON+=("DIST_DIR=${GKI_DIST_DIR}") + ( env -i bash -c "source ${OLD_ENVIRONMENT}; rm -f ${OLD_ENVIRONMENT}; export ${GKI_ENVIRON[*]} ; ./build/build.sh" ) || exit 1 + + # Dist dir must have vmlinux.symvers, modules.builtin.modinfo, modules.builtin + MAKE_ARGS+=("KBUILD_MIXED_TREE=$(readlink -m ${GKI_DIST_DIR})") +else + rm -f ${OLD_ENVIRONMENT} +fi + +BOOT_IMAGE_HEADER_VERSION=${BOOT_IMAGE_HEADER_VERSION:-3} + +cd ${ROOT_DIR} + +export CLANG_TRIPLE CROSS_COMPILE CROSS_COMPILE_COMPAT CROSS_COMPILE_ARM32 ARCH SUBARCH MAKE_GOALS + +# Restore the previously saved CC argument that might have been overridden by +# the BUILD_CONFIG. +[ -n "${CC_ARG}" ] && CC="${CC_ARG}" + +# CC=gcc is effectively a fallback to the default gcc including any target +# triplets. An absolute path (e.g., CC=/usr/bin/gcc) must be specified to use a +# custom compiler. +[ "${CC}" == "gcc" ] && unset CC && unset CC_ARG + +TOOL_ARGS=() + +# LLVM=1 implies what is otherwise set below; it is a more concise way of +# specifying CC=clang LD=ld.lld NM=llvm-nm OBJCOPY=llvm-objcopy , for +# newer kernel versions. +if [[ -n "${LLVM}" ]]; then + TOOL_ARGS+=("LLVM=1") + # Reset a bunch of variables that the kernel's top level Makefile does, just + # in case someone tries to use these binaries in this script such as in + # initramfs generation below. + HOSTCC=clang + HOSTCXX=clang++ + CC=clang + LD=ld.lld + AR=llvm-ar + NM=llvm-nm + OBJCOPY=llvm-objcopy + OBJDUMP=llvm-objdump + READELF=llvm-readelf + OBJSIZE=llvm-size + STRIP=llvm-strip +else + if [ -n "${HOSTCC}" ]; then + TOOL_ARGS+=("HOSTCC=${HOSTCC}") + fi + + if [ -n "${CC}" ]; then + TOOL_ARGS+=("CC=${CC}") + if [ -z "${HOSTCC}" ]; then + TOOL_ARGS+=("HOSTCC=${CC}") + fi + fi + + if [ -n "${LD}" ]; then + TOOL_ARGS+=("LD=${LD}" "HOSTLD=${LD}") + fi + + if [ -n "${NM}" ]; then + TOOL_ARGS+=("NM=${NM}") + fi + + if [ -n "${OBJCOPY}" ]; then + TOOL_ARGS+=("OBJCOPY=${OBJCOPY}") + fi +fi + +if [ -n "${LLVM_IAS}" ]; then + TOOL_ARGS+=("LLVM_IAS=${LLVM_IAS}") + # Reset $AS for the same reason that we reset $CC etc above. + AS=clang +fi + +if [ -n "${DEPMOD}" ]; then + TOOL_ARGS+=("DEPMOD=${DEPMOD}") +fi + +if [ -n "${DTC}" ]; then + TOOL_ARGS+=("DTC=${DTC}") +fi + +# Allow hooks that refer to $CC_LD_ARG to keep working until they can be +# updated. +CC_LD_ARG="${TOOL_ARGS[@]}" + +DECOMPRESS_GZIP="gzip -c -d" +DECOMPRESS_LZ4="lz4 -c -d -l" +if [ -z "${LZ4_RAMDISK}" ] ; then + RAMDISK_COMPRESS="gzip -c -f" + RAMDISK_DECOMPRESS="${DECOMPRESS_GZIP}" + RAMDISK_EXT="gz" +else + RAMDISK_COMPRESS="lz4 -c -l -12 --favor-decSpeed" + RAMDISK_DECOMPRESS="${DECOMPRESS_LZ4}" + RAMDISK_EXT="lz4" +fi + +if [ -n "${SKIP_IF_VERSION_MATCHES}" ]; then + if [ -f "${DIST_DIR}/vmlinux" ]; then + kernelversion="$(cd ${KERNEL_DIR} && make -s "${TOOL_ARGS[@]}" O=${OUT_DIR} kernelrelease)" + # Split grep into 2 steps. "Linux version" will always be towards top and fast to find. Don't + # need to search the entire vmlinux for it + if [[ ! "$kernelversion" =~ .*dirty.* ]] && \ + grep -o -a -m1 "Linux version [^ ]* " ${DIST_DIR}/vmlinux | grep -q " ${kernelversion} " ; then + echo "========================================================" + echo " Skipping build because kernel version matches ${kernelversion}" + exit 0 + fi + fi +fi + +mkdir -p ${OUT_DIR} ${DIST_DIR} + +if [ -n "${GKI_PREBUILTS_DIR}" ]; then + echo "========================================================" + echo " Copying GKI prebuilts" + GKI_PREBUILTS_DIR=$(readlink -m ${GKI_PREBUILTS_DIR}) + if [ ! -d "${GKI_PREBUILTS_DIR}" ]; then + echo "ERROR: ${GKI_PREBULTS_DIR} does not exist." >&2 + exit 1 + fi + for file in ${GKI_PREBUILTS_DIR}/*; do + filename=$(basename ${file}) + if ! $(cmp -s ${file} ${DIST_DIR}/${filename}); then + cp -v ${file} ${DIST_DIR}/${filename} + fi + done + MAKE_ARGS+=("KBUILD_MIXED_TREE=${GKI_PREBUILTS_DIR}") +fi + +echo "========================================================" +echo " Setting up for build" +if [ "${SKIP_MRPROPER}" != "1" ] ; then + set -x + (cd ${KERNEL_DIR} && make "${TOOL_ARGS[@]}" O=${OUT_DIR} "${MAKE_ARGS[@]}" mrproper) + set +x +fi + +if [ -n "${PRE_DEFCONFIG_CMDS}" ]; then + echo "========================================================" + echo " Running pre-defconfig command(s):" + set -x + eval ${PRE_DEFCONFIG_CMDS} + set +x +fi + +if [ "${SKIP_DEFCONFIG}" != "1" ] ; then + set -x + (cd ${KERNEL_DIR} && make "${TOOL_ARGS[@]}" O=${OUT_DIR} "${MAKE_ARGS[@]}" ${DEFCONFIG}) + set +x + + if [ -n "${POST_DEFCONFIG_CMDS}" ]; then + echo "========================================================" + echo " Running pre-make command(s):" + set -x + eval ${POST_DEFCONFIG_CMDS} + set +x + fi +fi + +if [ "${LTO}" = "none" -o "${LTO}" = "thin" -o "${LTO}" = "full" ]; then + echo "========================================================" + echo " Modifying LTO mode to '${LTO}'" + + set -x + if [ "${LTO}" = "none" ]; then + ${KERNEL_DIR}/scripts/config --file ${OUT_DIR}/.config \ + -d LTO_CLANG \ + -e LTO_NONE \ + -d LTO_CLANG_THIN \ + -d LTO_CLANG_FULL \ + -d THINLTO + elif [ "${LTO}" = "thin" ]; then + # This is best-effort; some kernels don't support LTO_THIN mode + # THINLTO was the old name for LTO_THIN, and it was 'default y' + ${KERNEL_DIR}/scripts/config --file ${OUT_DIR}/.config \ + -e LTO_CLANG \ + -d LTO_NONE \ + -e LTO_CLANG_THIN \ + -d LTO_CLANG_FULL \ + -e THINLTO + elif [ "${LTO}" = "full" ]; then + # THINLTO was the old name for LTO_THIN, and it was 'default y' + ${KERNEL_DIR}/scripts/config --file ${OUT_DIR}/.config \ + -e LTO_CLANG \ + -d LTO_NONE \ + -d LTO_CLANG_THIN \ + -e LTO_CLANG_FULL \ + -d THINLTO + fi + (cd ${OUT_DIR} && make "${TOOL_ARGS[@]}" O=${OUT_DIR} "${MAKE_ARGS[@]}" olddefconfig) + set +x +elif [ -n "${LTO}" ]; then + echo "LTO= must be one of 'none', 'thin' or 'full'." + exit 1 +fi + +if [ -n "${TAGS_CONFIG}" ]; then + echo "========================================================" + echo " Running tags command:" + set -x + (cd ${KERNEL_DIR} && SRCARCH=${ARCH} ./scripts/tags.sh ${TAGS_CONFIG}) + set +x + exit 0 +fi + +# Truncate abi.prop file +ABI_PROP=${DIST_DIR}/abi.prop +: > ${ABI_PROP} + +if [ -n "${ABI_DEFINITION}" ]; then + + ABI_XML=${DIST_DIR}/abi.xml + + echo "KMI_DEFINITION=abi.xml" >> ${ABI_PROP} + echo "KMI_MONITORED=1" >> ${ABI_PROP} + + if [ "${KMI_ENFORCED}" = "1" ]; then + echo "KMI_ENFORCED=1" >> ${ABI_PROP} + fi +fi + +if [ -n "${KMI_SYMBOL_LIST}" ]; then + ABI_SL=${DIST_DIR}/abi_symbollist + echo "KMI_SYMBOL_LIST=abi_symbollist" >> ${ABI_PROP} +fi + +# define the kernel binary and modules archive in the $ABI_PROP +echo "KERNEL_BINARY=vmlinux" >> ${ABI_PROP} +if [ "${COMPRESS_UNSTRIPPED_MODULES}" = "1" ]; then + echo "MODULES_ARCHIVE=${UNSTRIPPED_MODULES_ARCHIVE}" >> ${ABI_PROP} +fi + +# Copy the abi_${arch}.xml file from the sources into the dist dir +if [ -n "${ABI_DEFINITION}" ]; then + echo "========================================================" + echo " Copying abi definition to ${ABI_XML}" + pushd $ROOT_DIR/$KERNEL_DIR + cp "${ABI_DEFINITION}" ${ABI_XML} + popd +fi + +# Copy the abi symbol list file from the sources into the dist dir +if [ -n "${KMI_SYMBOL_LIST}" ]; then + ${ROOT_DIR}/build/copy_symbols.sh "$ABI_SL" "$ROOT_DIR/$KERNEL_DIR" \ + "${KMI_SYMBOL_LIST}" ${ADDITIONAL_KMI_SYMBOL_LISTS} + pushd $ROOT_DIR/$KERNEL_DIR + if [ "${TRIM_NONLISTED_KMI}" = "1" ]; then + # Create the raw symbol list + cat ${ABI_SL} | \ + ${ROOT_DIR}/build/abi/flatten_symbol_list > \ + ${OUT_DIR}/abi_symbollist.raw + + # Update the kernel configuration + ./scripts/config --file ${OUT_DIR}/.config \ + -d UNUSED_SYMBOLS -e TRIM_UNUSED_KSYMS \ + --set-str UNUSED_KSYMS_WHITELIST ${OUT_DIR}/abi_symbollist.raw + (cd ${OUT_DIR} && \ + make O=${OUT_DIR} "${TOOL_ARGS[@]}" "${MAKE_ARGS[@]}" olddefconfig) + # Make sure the config is applied + grep CONFIG_UNUSED_KSYMS_WHITELIST ${OUT_DIR}/.config > /dev/null || { + echo "ERROR: Failed to apply TRIM_NONLISTED_KMI kernel configuration" >&2 + echo "Does your kernel support CONFIG_UNUSED_KSYMS_WHITELIST?" >&2 + exit 1 + } + + elif [ "${KMI_SYMBOL_LIST_STRICT_MODE}" = "1" ]; then + echo "ERROR: KMI_SYMBOL_LIST_STRICT_MODE requires TRIM_NONLISTED_KMI=1" >&2 + exit 1 + fi + popd # $ROOT_DIR/$KERNEL_DIR +elif [ "${TRIM_NONLISTED_KMI}" = "1" ]; then + echo "ERROR: TRIM_NONLISTED_KMI requires a KMI_SYMBOL_LIST" >&2 + exit 1 +elif [ "${KMI_SYMBOL_LIST_STRICT_MODE}" = "1" ]; then + echo "ERROR: KMI_SYMBOL_LIST_STRICT_MODE requires a KMI_SYMBOL_LIST" >&2 + exit 1 +fi + +echo "========================================================" +echo " Building kernel" + +set -x +(cd ${OUT_DIR} && make O=${OUT_DIR} "${TOOL_ARGS[@]}" "${MAKE_ARGS[@]}" ${MAKE_GOALS}) +set +x + +if [ -n "${POST_KERNEL_BUILD_CMDS}" ]; then + echo "========================================================" + echo " Running post-kernel-build command(s):" + set -x + eval ${POST_KERNEL_BUILD_CMDS} + set +x +fi + +if [ -n "${MODULES_ORDER}" ]; then + echo "========================================================" + echo " Checking the list of modules:" + if ! diff -u "${KERNEL_DIR}/${MODULES_ORDER}" "${OUT_DIR}/modules.order"; then + echo "ERROR: modules list out of date" >&2 + echo "Update it with:" >&2 + echo "cp ${OUT_DIR}/modules.order ${KERNEL_DIR}/${MODULES_ORDER}" >&2 + exit 1 + fi +fi + +if [ "${KMI_SYMBOL_LIST_STRICT_MODE}" = "1" ]; then + echo "========================================================" + echo " Comparing the KMI and the symbol lists:" + set -x + ${ROOT_DIR}/build/abi/compare_to_symbol_list "${OUT_DIR}/Module.symvers" \ + "${OUT_DIR}/abi_symbollist.raw" + set +x +fi + +rm -rf ${MODULES_STAGING_DIR} +mkdir -p ${MODULES_STAGING_DIR} + +if [ "${DO_NOT_STRIP_MODULES}" != "1" ]; then + MODULE_STRIP_FLAG="INSTALL_MOD_STRIP=1" +fi + +if [ "${BUILD_INITRAMFS}" = "1" -o -n "${IN_KERNEL_MODULES}" ]; then + echo "========================================================" + echo " Installing kernel modules into staging directory" + + (cd ${OUT_DIR} && \ + make O=${OUT_DIR} "${TOOL_ARGS[@]}" ${MODULE_STRIP_FLAG} \ + INSTALL_MOD_PATH=${MODULES_STAGING_DIR} "${MAKE_ARGS[@]}" modules_install) +fi + +if [[ -z "${SKIP_EXT_MODULES}" ]] && [[ -n "${EXT_MODULES}" ]]; then + echo "========================================================" + echo " Building external modules and installing them into staging directory" + + for EXT_MOD in ${EXT_MODULES}; do + # The path that we pass in via the variable M needs to be a relative path + # relative to the kernel source directory. The source files will then be + # looked for in ${KERNEL_DIR}/${EXT_MOD_REL} and the object files (i.e. .o + # and .ko) files will be stored in ${OUT_DIR}/${EXT_MOD_REL}. If we + # instead set M to an absolute path, then object (i.e. .o and .ko) files + # are stored in the module source directory which is not what we want. + EXT_MOD_REL=$(rel_path ${ROOT_DIR}/${EXT_MOD} ${KERNEL_DIR}) + # The output directory must exist before we invoke make. Otherwise, the + # build system behaves horribly wrong. + mkdir -p ${OUT_DIR}/${EXT_MOD_REL} + set -x + make -C ${EXT_MOD} M=${EXT_MOD_REL} KERNEL_SRC=${ROOT_DIR}/${KERNEL_DIR} \ + O=${OUT_DIR} "${TOOL_ARGS[@]}" "${MAKE_ARGS[@]}" + make -C ${EXT_MOD} M=${EXT_MOD_REL} KERNEL_SRC=${ROOT_DIR}/${KERNEL_DIR} \ + O=${OUT_DIR} "${TOOL_ARGS[@]}" ${MODULE_STRIP_FLAG} \ + INSTALL_MOD_PATH=${MODULES_STAGING_DIR} \ + "${MAKE_ARGS[@]}" modules_install + set +x + done + +fi + +if [ -n "${EXTRA_CMDS}" ]; then + echo "========================================================" + echo " Running extra build command(s):" + set -x + eval ${EXTRA_CMDS} + set +x +fi + +OVERLAYS_OUT="" +for ODM_DIR in ${ODM_DIRS}; do + OVERLAY_DIR=${ROOT_DIR}/device/${ODM_DIR}/overlays + + if [ -d ${OVERLAY_DIR} ]; then + OVERLAY_OUT_DIR=${OUT_DIR}/overlays/${ODM_DIR} + mkdir -p ${OVERLAY_OUT_DIR} + make -C ${OVERLAY_DIR} DTC=${OUT_DIR}/scripts/dtc/dtc \ + OUT_DIR=${OVERLAY_OUT_DIR} "${MAKE_ARGS[@]}" + OVERLAYS=$(find ${OVERLAY_OUT_DIR} -name "*.dtbo") + OVERLAYS_OUT="$OVERLAYS_OUT $OVERLAYS" + fi +done + +echo "========================================================" +echo " Copying files" +for FILE in $(cd ${OUT_DIR} && ls -1 ${FILES}); do + if [ -f ${OUT_DIR}/${FILE} ]; then + echo " $FILE" + cp -p ${OUT_DIR}/${FILE} ${DIST_DIR}/ + else + echo " $FILE is not a file, skipping" + fi +done + +for FILE in ${OVERLAYS_OUT}; do + OVERLAY_DIST_DIR=${DIST_DIR}/$(dirname ${FILE#${OUT_DIR}/overlays/}) + echo " ${FILE#${OUT_DIR}/}" + mkdir -p ${OVERLAY_DIST_DIR} + cp ${FILE} ${OVERLAY_DIST_DIR}/ +done + +if [ -z "${SKIP_CP_KERNEL_HDR}" ]; then + echo "========================================================" + echo " Installing UAPI kernel headers:" + mkdir -p "${KERNEL_UAPI_HEADERS_DIR}/usr" + make -C ${OUT_DIR} O=${OUT_DIR} "${TOOL_ARGS[@]}" \ + INSTALL_HDR_PATH="${KERNEL_UAPI_HEADERS_DIR}/usr" "${MAKE_ARGS[@]}" \ + headers_install + # The kernel makefiles create files named ..install.cmd and .install which + # are only side products. We don't want those. Let's delete them. + find ${KERNEL_UAPI_HEADERS_DIR} \( -name ..install.cmd -o -name .install \) -exec rm '{}' + + KERNEL_UAPI_HEADERS_TAR=${DIST_DIR}/kernel-uapi-headers.tar.gz + echo " Copying kernel UAPI headers to ${KERNEL_UAPI_HEADERS_TAR}" + tar -czf ${KERNEL_UAPI_HEADERS_TAR} --directory=${KERNEL_UAPI_HEADERS_DIR} usr/ +fi + +if [ -z "${SKIP_CP_KERNEL_HDR}" ] ; then + echo "========================================================" + KERNEL_HEADERS_TAR=${DIST_DIR}/kernel-headers.tar.gz + echo " Copying kernel headers to ${KERNEL_HEADERS_TAR}" + pushd $ROOT_DIR/$KERNEL_DIR + find arch include $OUT_DIR -name *.h -print0 \ + | tar -czf $KERNEL_HEADERS_TAR \ + --absolute-names \ + --dereference \ + --transform "s,.*$OUT_DIR,," \ + --transform "s,^,kernel-headers/," \ + --null -T - + popd +fi + +if [ "${GENERATE_VMLINUX_BTF}" = "1" ]; then + echo "========================================================" + echo " Generating ${DIST_DIR}/vmlinux.btf" + + ( + cd ${DIST_DIR} + cp -a vmlinux vmlinux.btf + pahole -J vmlinux.btf + llvm-strip --strip-debug vmlinux.btf + ) + +fi + +if [ -n "${GKI_DIST_DIR}" ]; then + echo "========================================================" + echo " Copying files from GKI kernel" + cp -rv ${GKI_DIST_DIR}/* ${DIST_DIR}/ +fi + +if [ -n "${DIST_CMDS}" ]; then + echo "========================================================" + echo " Running extra dist command(s):" + # if DIST_CMDS requires UAPI headers, make sure a warning appears! + if [ ! -d "${KERNEL_UAPI_HEADERS_DIR}/usr" ]; then + echo "WARN: running without UAPI headers" + fi + set -x + eval ${DIST_CMDS} + set +x +fi + +MODULES=$(find ${MODULES_STAGING_DIR} -type f -name "*.ko") +if [ -n "${MODULES}" ]; then + if [ -n "${IN_KERNEL_MODULES}" -o -n "${EXT_MODULES}" ]; then + echo "========================================================" + echo " Copying modules files" + for FILE in ${MODULES}; do + echo " ${FILE#${MODULES_STAGING_DIR}/}" + cp -p ${FILE} ${DIST_DIR} + done + fi + if [ "${BUILD_INITRAMFS}" = "1" ]; then + echo "========================================================" + echo " Creating initramfs" + rm -rf ${INITRAMFS_STAGING_DIR} + create_modules_staging "${MODULES_LIST}" ${MODULES_STAGING_DIR} \ + ${INITRAMFS_STAGING_DIR} "${MODULES_BLOCKLIST}" "-e" + + MODULES_ROOT_DIR=$(echo ${INITRAMFS_STAGING_DIR}/lib/modules/*) + cp ${MODULES_ROOT_DIR}/modules.load ${DIST_DIR}/modules.load + cp ${MODULES_ROOT_DIR}/modules.load ${DIST_DIR}/vendor_boot.modules.load + echo "${MODULES_OPTIONS}" > ${MODULES_ROOT_DIR}/modules.options + + mkbootfs "${INITRAMFS_STAGING_DIR}" >"${MODULES_STAGING_DIR}/initramfs.cpio" + ${RAMDISK_COMPRESS} "${MODULES_STAGING_DIR}/initramfs.cpio" >"${DIST_DIR}/initramfs.img" + fi +fi + +if [ -n "${VENDOR_DLKM_MODULES_LIST}" ]; then + build_vendor_dlkm +fi + +if [ -n "${UNSTRIPPED_MODULES}" ]; then + echo "========================================================" + echo " Copying unstripped module files for debugging purposes (not loaded on device)" + mkdir -p ${UNSTRIPPED_DIR} + for MODULE in ${UNSTRIPPED_MODULES}; do + find ${MODULES_PRIVATE_DIR} -name ${MODULE} -exec cp {} ${UNSTRIPPED_DIR} \; + done + if [ "${COMPRESS_UNSTRIPPED_MODULES}" = "1" ]; then + tar -czf ${DIST_DIR}/${UNSTRIPPED_MODULES_ARCHIVE} -C $(dirname ${UNSTRIPPED_DIR}) $(basename ${UNSTRIPPED_DIR}) + rm -rf ${UNSTRIPPED_DIR} + fi +fi + +[ -n "${GKI_MODULES_LIST}" ] && cp ${KERNEL_DIR}/${GKI_MODULES_LIST} ${DIST_DIR}/ + +echo "========================================================" +echo " Files copied to ${DIST_DIR}" + +if [ -n "${BUILD_BOOT_IMG}" -o -n "${BUILD_VENDOR_BOOT_IMG}" ] ; then + if [ -z "${MKBOOTIMG_PATH}" ]; then + MKBOOTIMG_PATH="tools/mkbootimg/mkbootimg.py" + fi + if [ ! -f "${MKBOOTIMG_PATH}" ]; then + echo "mkbootimg.py script not found. MKBOOTIMG_PATH = ${MKBOOTIMG_PATH}" + exit 1 + fi + + MKBOOTIMG_ARGS=("--header_version" "${BOOT_IMAGE_HEADER_VERSION}") + if [ -n "${BASE_ADDRESS}" ]; then + MKBOOTIMG_ARGS+=("--base" "${BASE_ADDRESS}") + fi + if [ -n "${PAGE_SIZE}" ]; then + MKBOOTIMG_ARGS+=("--pagesize" "${PAGE_SIZE}") + fi + if [ -n "${KERNEL_VENDOR_CMDLINE}" -a "${BOOT_IMAGE_HEADER_VERSION}" -lt "3" ]; then + KERNEL_CMDLINE+=" ${KERNEL_VENDOR_CMDLINE}" + fi + if [ -n "${KERNEL_CMDLINE}" ]; then + MKBOOTIMG_ARGS+=("--cmdline" "${KERNEL_CMDLINE}") + fi + if [ -n "${TAGS_OFFSET}" ]; then + MKBOOTIMG_ARGS+=("--tags_offset" "${TAGS_OFFSET}") + fi + if [ -n "${RAMDISK_OFFSET}" ]; then + MKBOOTIMG_ARGS+=("--ramdisk_offset" "${RAMDISK_OFFSET}") + fi + + DTB_FILE_LIST=$(find ${DIST_DIR} -name "*.dtb" | sort) + if [ -z "${DTB_FILE_LIST}" ]; then + if [ -z "${SKIP_VENDOR_BOOT}" ]; then + echo "No *.dtb files found in ${DIST_DIR}" + exit 1 + fi + else + cat $DTB_FILE_LIST > ${DIST_DIR}/dtb.img + MKBOOTIMG_ARGS+=("--dtb" "${DIST_DIR}/dtb.img") + fi + + rm -rf "${MKBOOTIMG_STAGING_DIR}" + MKBOOTIMG_RAMDISK_STAGING_DIR="${MKBOOTIMG_STAGING_DIR}/ramdisk_root" + mkdir -p "${MKBOOTIMG_RAMDISK_STAGING_DIR}" + + if [ -n "${VENDOR_RAMDISK_BINARY}" ]; then + VENDOR_RAMDISK_CPIO="${MKBOOTIMG_STAGING_DIR}/vendor_ramdisk_binary.cpio" + rm -f "${VENDOR_RAMDISK_CPIO}" + for vendor_ramdisk_binary in ${VENDOR_RAMDISK_BINARY}; do + if ! [ -f "${vendor_ramdisk_binary}" ]; then + echo "Unable to locate vendor ramdisk ${vendor_ramdisk_binary}." + exit 1 + fi + if ${DECOMPRESS_GZIP} "${vendor_ramdisk_binary}" 2>/dev/null >> "${VENDOR_RAMDISK_CPIO}"; then + echo "${vendor_ramdisk_binary} is GZIP compressed" + elif ${DECOMPRESS_LZ4} "${vendor_ramdisk_binary}" 2>/dev/null >> "${VENDOR_RAMDISK_CPIO}"; then + echo "${vendor_ramdisk_binary} is LZ4 compressed" + elif cpio -t < "${vendor_ramdisk_binary}" &>/dev/null; then + echo "${vendor_ramdisk_binary} is plain CPIO archive" + cat "${vendor_ramdisk_binary}" >> "${VENDOR_RAMDISK_CPIO}" + else + echo "Unable to identify type of vendor ramdisk ${vendor_ramdisk_binary}" + rm -f "${VENDOR_RAMDISK_CPIO}" + exit 1 + fi + done + + # Remove lib/modules from the vendor ramdisk binary + # Also execute ${VENDOR_RAMDISK_CMDS} for further modifications + ( cd "${MKBOOTIMG_RAMDISK_STAGING_DIR}" + cpio -idu --quiet <"${VENDOR_RAMDISK_CPIO}" + rm -rf lib/modules + eval ${VENDOR_RAMDISK_CMDS} + ) + fi + + if [ -f "${VENDOR_FSTAB}" ]; then + mkdir -p "${MKBOOTIMG_RAMDISK_STAGING_DIR}/first_stage_ramdisk" + cp "${VENDOR_FSTAB}" "${MKBOOTIMG_RAMDISK_STAGING_DIR}/first_stage_ramdisk/" + fi + + HAS_RAMDISK= + MKBOOTIMG_RAMDISK_DIRS=() + if [ -n "${VENDOR_RAMDISK_BINARY}" ] || [ -f "${VENDOR_FSTAB}" ]; then + HAS_RAMDISK="1" + MKBOOTIMG_RAMDISK_DIRS+=("${MKBOOTIMG_RAMDISK_STAGING_DIR}") + fi + + if [ "${BUILD_INITRAMFS}" = "1" ]; then + HAS_RAMDISK="1" + if [ -z "${INITRAMFS_VENDOR_RAMDISK_FRAGMENT_NAME}" ]; then + MKBOOTIMG_RAMDISK_DIRS+=("${INITRAMFS_STAGING_DIR}") + fi + fi + + if [ -z "${HAS_RAMDISK}" ] && [ -z "${SKIP_VENDOR_BOOT}" ]; then + echo "No ramdisk found. Please provide a GKI and/or a vendor ramdisk." + exit 1 + fi + + if [ "${#MKBOOTIMG_RAMDISK_DIRS[@]}" -gt 0 ]; then + MKBOOTIMG_RAMDISK_CPIO="${MKBOOTIMG_STAGING_DIR}/ramdisk.cpio" + mkbootfs "${MKBOOTIMG_RAMDISK_DIRS[@]}" >"${MKBOOTIMG_RAMDISK_CPIO}" + ${RAMDISK_COMPRESS} "${MKBOOTIMG_RAMDISK_CPIO}" >"${DIST_DIR}/ramdisk.${RAMDISK_EXT}" + fi + + if [ -n "${BUILD_BOOT_IMG}" ]; then + if [ ! -f "${DIST_DIR}/$KERNEL_BINARY" ]; then + echo "kernel binary(KERNEL_BINARY = $KERNEL_BINARY) not present in ${DIST_DIR}" + exit 1 + fi + MKBOOTIMG_ARGS+=("--kernel" "${DIST_DIR}/${KERNEL_BINARY}") + fi + + if [ "${BOOT_IMAGE_HEADER_VERSION}" -ge "4" ]; then + if [ -n "${VENDOR_BOOTCONFIG}" ]; then + for PARAM in ${VENDOR_BOOTCONFIG}; do + echo "${PARAM}" + done >"${DIST_DIR}/vendor-bootconfig.img" + MKBOOTIMG_ARGS+=("--vendor_bootconfig" "${DIST_DIR}/vendor-bootconfig.img") + KERNEL_VENDOR_CMDLINE+=" bootconfig" + fi + fi + + if [ "${BOOT_IMAGE_HEADER_VERSION}" -ge "3" ]; then + if [ -f "${GKI_RAMDISK_PREBUILT_BINARY}" ]; then + MKBOOTIMG_ARGS+=("--ramdisk" "${GKI_RAMDISK_PREBUILT_BINARY}") + fi + + if [ -z "${SKIP_VENDOR_BOOT}" ]; then + MKBOOTIMG_ARGS+=("--vendor_boot" "${DIST_DIR}/vendor_boot.img") + if [ -n "${KERNEL_VENDOR_CMDLINE}" ]; then + MKBOOTIMG_ARGS+=("--vendor_cmdline" "${KERNEL_VENDOR_CMDLINE}") + fi + if [ -f "${DIST_DIR}/ramdisk.${RAMDISK_EXT}" ]; then + MKBOOTIMG_ARGS+=("--vendor_ramdisk" "${DIST_DIR}/ramdisk.${RAMDISK_EXT}") + fi + if [ "${BUILD_INITRAMFS}" = "1" ] \ + && [ -n "${INITRAMFS_VENDOR_RAMDISK_FRAGMENT_NAME}" ]; then + MKBOOTIMG_ARGS+=("--ramdisk_type" "DLKM") + for MKBOOTIMG_ARG in ${INITRAMFS_VENDOR_RAMDISK_FRAGMENT_MKBOOTIMG_ARGS}; do + MKBOOTIMG_ARGS+=("${MKBOOTIMG_ARG}") + done + MKBOOTIMG_ARGS+=("--ramdisk_name" "${INITRAMFS_VENDOR_RAMDISK_FRAGMENT_NAME}") + MKBOOTIMG_ARGS+=("--vendor_ramdisk_fragment" "${DIST_DIR}/initramfs.img") + fi + fi + else + if [ -f "${DIST_DIR}/ramdisk.${RAMDISK_EXT}" ]; then + MKBOOTIMG_ARGS+=("--ramdisk" "${DIST_DIR}/ramdisk.${RAMDISK_EXT}") + fi + fi + + if [ -n "${BUILD_BOOT_IMG}" ]; then + MKBOOTIMG_ARGS+=("--output" "${DIST_DIR}/boot.img") + fi + + for MKBOOTIMG_ARG in ${MKBOOTIMG_EXTRA_ARGS}; do + MKBOOTIMG_ARGS+=("${MKBOOTIMG_ARG}") + done + + "${MKBOOTIMG_PATH}" "${MKBOOTIMG_ARGS[@]}" + + if [ -n "${BUILD_BOOT_IMG}" -a -f "${DIST_DIR}/boot.img" ]; then + echo "boot image created at ${DIST_DIR}/boot.img" + + if [ -n "${AVB_SIGN_BOOT_IMG}" ]; then + if [ -n "${AVB_BOOT_PARTITION_SIZE}" ] \ + && [ -n "${AVB_BOOT_KEY}" ] \ + && [ -n "${AVB_BOOT_ALGORITHM}" ]; then + echo "Signing the boot.img..." + avbtool add_hash_footer --partition_name boot \ + --partition_size ${AVB_BOOT_PARTITION_SIZE} \ + --image ${DIST_DIR}/boot.img \ + --algorithm ${AVB_BOOT_ALGORITHM} \ + --key ${AVB_BOOT_KEY} + else + echo "Missing the AVB_* flags. Failed to sign the boot image" 1>&2 + exit 1 + fi + fi + fi + + [ -z "${SKIP_VENDOR_BOOT}" ] \ + && [ "${BOOT_IMAGE_HEADER_VERSION}" -ge "3" ] \ + && [ -f "${DIST_DIR}/vendor_boot.img" ] \ + && echo "vendor boot image created at ${DIST_DIR}/vendor_boot.img" +fi + + +# No trace_printk use on build server build +if readelf -a ${DIST_DIR}/vmlinux 2>&1 | grep -q trace_printk_fmt; then + echo "========================================================" + echo "WARN: Found trace_printk usage in vmlinux." + echo "" + echo "trace_printk will cause trace_printk_init_buffers executed in kernel" + echo "start, which will increase memory and lead warning shown during boot." + echo "We should not carry trace_printk in production kernel." + echo "" + if [ ! -z "${STOP_SHIP_TRACEPRINTK}" ]; then + echo "ERROR: stop ship on trace_printk usage." 1>&2 + exit 1 + fi +fi diff --git a/build_abi.sh b/build_abi.sh new file mode 100755 index 0000000000..a81dd2733a --- /dev/null +++ b/build_abi.sh @@ -0,0 +1,377 @@ +#!/bin/bash + +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Usage: +# build/build_abi.sh +# +# The following environment variables are considered during execution: +# +# ABI_OUT_TAG +# Customize the output file name for the abi dump. If undefined, the tag is +# derived from `git describe`. +# +# ABI_DEFINITION +# Specify an expected Kernel ABI representation. If defined, this script +# will, in addition to extracting the ABI representation from the currently +# built kernel, compare the extracted ABI to the expected one. In case of +# any significant differences, it will exit with the return code of +# diff_abi and optionally (-r) print a report. +# ABI_DEFINITION is supposed to be defined relative to $KERNEL_DIR/ +# +# TIDY_ABI +# If set to 1, any ABI dump created will be post-processed by the abitidy +# utility. This is done to minimise the XML and XML diffs. The utility +# removes unreachable parts of the ABI XML, sorts its elements into a +# canonical order and gives anonymous types consistent internal names. +# +# KMI_SYMBOL_LIST +# Define a Kernel Module Interface symbol list description. If defined, it +# will be taken into account when extracting Kernel ABI information from +# vmlinux and kernel modules. +# KMI_SYMBOL_LIST is supposed to be defined relative to $KERNEL_DIR/ +# +# KMI_SYMBOL_LIST_MODULE_GROUPING +# If set to 1, then the symbol list will group symbols based on the kernel +# modules that reference the symbol. Otherwise the symbol list will simply +# be a sorted list of symbols used by all the kernel modules. This property +# is enabled by default. +# +# KMI_SYMBOL_LIST_ADD_ONLY +# If set to 1, then any symbols in the symbol list that would have been +# removed are preserved (at the end of the file). Symbol list update will +# fail if there is no pre-existing symbol list file to read from. This +# property is intended to prevent unintentional shrinkage of a stable ABI. +# It is disabled by default. +# +# GKI_MODULES_LIST +# If set to a file name, then this file will be read to determine the list +# of GKI modules (those subject to ABI monitoring) and, by elimination, the +# list of vendor modules (those which can rely on a stable ABI). Only vendor +# modules' undefined symbols are considered when updating the symbol list. +# GKI_MODULES_LIST is supposed to be defined relative to $KERNEL_DIR/ +# +# FULL_GKI_ABI +# If this is set to 1 then, when updating the symbol list, use all defined +# symbols from vmlinux and GKI modules, instead of the undefined symbols +# from vendor modules. This property is disabled by default. + +export ROOT_DIR=$(readlink -f $(dirname $0)/..) + +function show_help { + echo "USAGE: $0 [-u|--update] [-n|--nodiff]" + echo + echo " -u | --update Update ABI representation and main symbol list in the source directory" + echo " -s | --update-symbol-list Update main symbol list in the source directory" + echo " -n | --nodiff Do not generate an ABI report with diff_abi" + echo " -r | --print-report Print ABI short report in case of any differences" + echo " -a | --full-report Create a detailed ABI report" +} + +UPDATE=0 +UPDATE_SYMBOL_LIST=0 +DIFF=1 +PRINT_REPORT=0 +FULL_REPORT=0 + +if [[ -z "${KMI_SYMBOL_LIST_MODULE_GROUPING}" ]]; then + KMI_SYMBOL_LIST_MODULE_GROUPING=1 +fi +if [[ -z "$KMI_SYMBOL_LIST_ADD_ONLY" ]]; then + KMI_SYMBOL_LIST_ADD_ONLY=0 +fi +if [[ -z "$FULL_GKI_ABI" ]]; then + FULL_GKI_ABI=0 +fi +if [[ -z "$TIDY_ABI" ]]; then + TIDY_ABI=0 +fi + +ARGS=() +for i in "$@" +do +case $i in + -u|--update) + UPDATE=1 + shift # past argument=value + ;; + -s|--update-symbol-list) + UPDATE_SYMBOL_LIST=1 + shift # past argument=value + ;; + -n|--nodiff) + DIFF=0 + shift # past argument=value + ;; + -r|--print-report) + PRINT_REPORT=1 + shift # past argument=value + ;; + -a|--full-report) + FULL_REPORT=1 + shift # past argument=value + ;; + -h|--help) + show_help + exit 0 + ;; + *) + ARGS+=("$1") + shift + ;; +esac +done + +set -- "${ARGS[@]}" + +set -e +set -a + +# if we are using the default OUT_DIR, add a suffix so we are free to wipe it +# before building to ensure a clean build/analysis. That is the default case. +if [[ -z "$OUT_DIR" ]]; then + export OUT_DIR_SUFFIX="_abi" + wipe_out_dir=1 +fi + +source "${ROOT_DIR}/build/_setup_env.sh" + +if [ -z "${KMI_SYMBOL_LIST}" ]; then + if [ $UPDATE_SYMBOL_LIST -eq 1 ]; then + echo "ERROR: --update-symbol-list requires a KMI_SYMBOL_LIST" >&2 + exit 1 + fi +elif [ $UPDATE -eq 1 ]; then + UPDATE_SYMBOL_LIST=1 +fi + +# Now actually do the wipe out as above. +if [[ $wipe_out_dir -eq 1 && -d ${COMMON_OUT_DIR} ]]; then + find "${COMMON_OUT_DIR}" \( -name 'vmlinux' -o -name '*.ko' \) -delete -print +fi + +# assert CONFIG_DEBUG_INFO=y +append_cmd POST_DEFCONFIG_CMDS 'check_config_for_abi_dump' +export POST_DEFCONFIG_CMDS +function check_config_for_abi_dump() { + local debug=$(${KERNEL_DIR}/scripts/config --file ${OUT_DIR}/.config -s DEBUG_INFO) + if [ "$debug" != y ]; then + echo "ERROR: DEBUG_INFO is not set in config" >&2 + exit 1 + fi +} +export -f check_config_for_abi_dump + +# Disable mixed build when comparing ABI snapshots. Device kernel ABI should compared, even in a +# mixed build environment. +GKI_BUILD_CONFIG= +# Mixed build device kernels would not compile vmlinux. When using build_abi.sh to compile, we +# do want to compile vmlinux since we are comparing the ABI of the device kernel. +MAKE_GOALS+=" vmlinux" + +function build_kernel() { + # Delegate the actual build to build.sh. + # Suppress possible values of ABI_DEFINITION when invoking build.sh to avoid + # the generated abi.xml to be copied to /abi.out. + # Turn on symtypes generation to assist in the diagnosis of CRC differences. + ABI_DEFINITION= \ + KBUILD_SYMTYPES=1 \ + ${ROOT_DIR}/build/build.sh "$@" +} + +# define a common KMI symbol list flag for the abi tools +KMI_SYMBOL_LIST_FLAG= + +# define a flag control ABI tidying +TIDY_ABI_FLAG= +if [ "$TIDY_ABI" -eq 1 ]; then + TIDY_ABI_FLAG=--tidy +fi + +# We want to track whether the main symbol list (i.e. KMI_SYMBOL_LIST) actually +# got updated. If so we need to rerun the kernel build. +if [ -n "$KMI_SYMBOL_LIST" ]; then + + if [ $UPDATE_SYMBOL_LIST -eq 1 ]; then + # Disable KMI trimming as the symbol list may be out of date. + TRIM_NONLISTED_KMI= KMI_SYMBOL_LIST_STRICT_MODE= build_kernel "$@" + + echo "========================================================" + echo " Updating the ABI symbol list" + + # Exclude GKI modules from non-GKI builds + if [ -n "${GKI_MODULES_LIST}" ]; then + GKI_MOD_FLAG="--gki-modules ${DIST_DIR}/$(basename ${GKI_MODULES_LIST})" + fi + if [ "$KMI_SYMBOL_LIST_ADD_ONLY" -eq 1 ]; then + ADD_ONLY_FLAG="--additions-only" + fi + # Specify a full GKI ABI if requested + if [ "$FULL_GKI_ABI" -eq 1 ]; then + FULL_ABI_FLAG="--full-gki-abi" + fi + + if [ "${KMI_SYMBOL_LIST_MODULE_GROUPING}" -eq "0" ]; then + SKIP_MODULE_GROUPING="--skip-module-grouping" + fi + + ${ROOT_DIR}/build/abi/extract_symbols \ + --symbol-list $KERNEL_DIR/$KMI_SYMBOL_LIST \ + ${SKIP_MODULE_GROUPING} \ + ${ADD_ONLY_FLAG} \ + ${GKI_MOD_FLAG} \ + ${FULL_ABI_FLAG} \ + ${DIST_DIR} + + # Redo what build.sh has done, with possibly fresher symbol lists. + ABI_SL="${DIST_DIR}/abi_symbollist" + ${ROOT_DIR}/build/copy_symbols.sh "$ABI_SL" "$ROOT_DIR/$KERNEL_DIR" \ + "${KMI_SYMBOL_LIST}" ${ADDITIONAL_KMI_SYMBOL_LISTS} + + # In case of a simple --update-symbol-list call we can bail out early + [ $UPDATE -eq 0 ] && exit 0 + + if [ -n "${TRIM_NONLISTED_KMI}" ]; then + # Rerun the kernel build with symbol list trimming enabled, as applicable. That + # influences the combined symbol list as well as the list of exported symbols in + # the kernel binary. Possibly more. + echo "========================================================" + echo " Rerunning the build with symbol trimming re-enabled" + SKIP_MRPROPER=1 + fi + fi + + KMI_SYMBOL_LIST_FLAG="--kmi-symbol-list ${DIST_DIR}/abi_symbollist" + +fi + +# Already built the final kernel if updating symbol list and trimming symbol list is disabled +if ! [ $UPDATE_SYMBOL_LIST -eq 1 -a -z "${TRIM_NONLISTED_KMI}" -a "$FULL_GKI_ABI" -eq 0 ]; then + SKIP_MRPROPER="${SKIP_MRPROPER}" build_kernel "$@" +fi + +echo "========================================================" +echo " Creating ABI dump" + +ABI_LINUX_TREE=${DIST_DIR} +ABI_VMLINUX_PATH= +DELETE_UNSTRIPPED_MODULES= +if [ -z "${DO_NOT_STRIP_MODULES}" ] && [ $(echo "${UNSTRIPPED_MODULES}" | tr -d '\n') = "*.ko" ]; then + if [ -n "${COMPRESS_UNSTRIPPED_MODULES}" ] && [ ! -f "${UNSTRIPPED_DIR}" ]; then + tar -xzf ${DIST_DIR}/${UNSTRIPPED_MODULES_ARCHIVE} -C $(dirname ${UNSTRIPPED_DIR}) + DELETE_UNSTRIPPED_MODULES=1 + fi + ABI_LINUX_TREE=${UNSTRIPPED_DIR} + ABI_VMLINUX_PATH="--vmlinux ${DIST_DIR}/vmlinux" +fi + +# create ABI dump +COMMON_OUT_DIR=$(readlink -m ${OUT_DIR:-${ROOT_DIR}/out/${BRANCH}}) +id=${ABI_OUT_TAG:-$(git -C $KERNEL_DIR describe --dirty --always)} +abi_out_file=abi-${id}.xml +full_abi_out_file=abi-full-${id}.xml +# if we have two to do, do them in parallel +${ROOT_DIR}/build/abi/dump_abi \ + --linux-tree ${ABI_LINUX_TREE} \ + ${ABI_VMLINUX_PATH} \ + --out-file ${DIST_DIR}/${abi_out_file} \ + $KMI_SYMBOL_LIST_FLAG $TIDY_ABI_FLAG & +if [ -n "$KMI_SYMBOL_LIST_FLAG" ]; then + ${ROOT_DIR}/build/abi/dump_abi \ + --linux-tree ${ABI_LINUX_TREE} \ + ${ABI_VMLINUX_PATH} \ + --out-file ${DIST_DIR}/${full_abi_out_file} \ + $TIDY_ABI_FLAG & + wait -n + wait -n +else + wait -n + cp ${DIST_DIR}/${abi_out_file} ${DIST_DIR}/${full_abi_out_file} +fi + +effective_kernel_dir=$(readlink -f ${ROOT_DIR}/${KERNEL_DIR}) +if [ -n "${LLVM}" ]; then + CC=clang +fi +for f in "$abi_out_file" "$full_abi_out_file"; do + # sanitize the abi.xml by removing any occurrences of the kernel path + # and also do that with any left over paths sneaking in + # (e.g. from the prebuilts) + sed -i -e "s#${effective_kernel_dir}/##g" \ + -e "s#${ROOT_DIR}/${KERNEL_DIR}/##g" \ + -e "s#${ROOT_DIR}/##g" "$DIST_DIR/$f" + # Append debug information to abi file + echo " +" >> ${DIST_DIR}/$f +done + +ln -sf ${abi_out_file} ${DIST_DIR}/abi.xml +ln -sf ${full_abi_out_file} ${DIST_DIR}/abi-full.xml +echo "========================================================" +echo " ABI dump has been created at ${DIST_DIR}/${abi_out_file}" +echo " Full ABI dump has been created at ${DIST_DIR}/${full_abi_out_file}" + +rc=0 +if [ -n "$ABI_DEFINITION" ]; then + if [ $DIFF -eq 1 ]; then + echo "========================================================" + echo " Comparing ABI against expected definition ($ABI_DEFINITION)" + abi_report=${DIST_DIR}/abi.report + + FULL_REPORT_FLAG= + if [ $FULL_REPORT -eq 1 ]; then + FULL_REPORT_FLAG="--full-report" + fi + + set +e + ${ROOT_DIR}/build/abi/diff_abi --baseline $KERNEL_DIR/$ABI_DEFINITION \ + --new ${DIST_DIR}/${abi_out_file} \ + --report ${abi_report} \ + --short-report ${abi_report}.short \ + $FULL_REPORT_FLAG + rc=$? + set -e + echo "========================================================" + echo " A brief ABI report has been created at ${abi_report}.short" + echo + echo " The detailed report is available in the same directory." + + if [ $rc -ne 0 ]; then + echo " ABI DIFFERENCES HAVE BEEN DETECTED! (RC=$rc)" 1>&2 + fi + + if [ $PRINT_REPORT -eq 1 ] && [ $rc -ne 0 ] ; then + echo "========================================================" 1>&2 + cat ${abi_report}.short 1>&2 + fi + fi + if [ $UPDATE -eq 1 ] ; then + echo "========================================================" + echo " Updating expected ABI definition ($ABI_DEFINITION)" + cp -v ${DIST_DIR}/${abi_out_file} $KERNEL_DIR/$ABI_DEFINITION + fi +fi + +[ -n "${DELETE_UNSTRIPPED_MODULES}" ] && rm -rf ${UNSTRIPPED_DIR} + +if [ -n "${KMI_ENFORCED}" ]; then + exit $rc +else + exit 0 +fi + diff --git a/build_test.sh b/build_test.sh new file mode 100755 index 0000000000..17adaf5f50 --- /dev/null +++ b/build_test.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Usage: +# build/build_test.sh + +export MAKE_ARGS=$@ +export ROOT_DIR=$(dirname $(readlink -f $0)) +export NET_TEST=${ROOT_DIR}/../kernel/tests/net/test + +# if device has its own build.config.net_test in the +# root (via manifest copy rule) then use it, otherwise +# use the default one in the build/ directory. if the +# BUILD_CONFIG is already specified in the environment, +# it overrides everything (unless it does not exist.) +if [ -z "$BUILD_CONFIG" ]; then + BUILD_CONFIG=build.config.net_test +fi +if [ ! -e $BUILD_CONFIG ]; then + BUILD_CONFIG=build/${BUILD_CONFIG} +fi +export BUILD_CONFIG + +test=all_tests.sh +set -e +source ${ROOT_DIR}/_setup_env.sh +export OUT_DIR=$(readlink -m ${OUT_DIR:-${ROOT_DIR}/out/${BRANCH}}) +mkdir -p ${OUT_DIR} + +# build.config.net_test sets KERNEL_DIR to "private/*", which doesn't work for +# common kernels, where the code is in "common/". Check for that here. We could +# also require that each of these kernels have their own build.config.net_test, +# but that complicates the manifests. +if ! [ -f $KERNEL_DIR/Makefile ] && [ -f common/Makefile ]; then + KERNEL_DIR=common +fi +export KERNEL_DIR=$(readlink -m ${KERNEL_DIR}) + +echo "========================================================" +echo " Building kernel and running tests " +echo " Using KERNEL_DIR: " ${KERNEL_DIR} +echo " Using OUT_DIR : " ${OUT_DIR} + +cd ${OUT_DIR} +$NET_TEST/run_net_test.sh --builder $test +echo $? +echo "======Finished running tests======" diff --git a/config.sh b/config.sh new file mode 100755 index 0000000000..cdc17ce0b7 --- /dev/null +++ b/config.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash + +# Usage: +# build/config.sh * +# +# Example: +# build/config.sh menuconfig|config|nconfig|... (default to menuconfig) +# +# Runs a configuration editor inside kernel/build environment. +# +# In addition to the environment variables considered in build/build.sh, the +# the following can be defined: +# +# FRAGMENT_CONFIG +# If set, the FRAGMENT_CONFIG file (absolute or relative to ROOT_DIR) is +# updated with the options selected by the config editor. +# +# Note: When editing a FRAGMENT_CONFIG, config.sh is intentionally +# unintelligent about removing "redundant" configuration options. That is, +# setting CONFIG_ARM_SMMU=m using config.sh, then unsetting it would +# result in a fragment config with CONFIG_ARM_SMMU explicitly unset. +# This behavior is desired since it is unknown whether the base +# configuration without the fragment would have CONFIG_ARM_SMMU (un)set. +# If desire is to let the base configuration properly control a CONFIG_ +# option, then remove the line from FRAGMENT_CONFIG + +export ROOT_DIR=$(readlink -f $(dirname $0)/..) + +# Disable hermetic toolchain for ncurses +# TODO: Support hermetic toolchain with ncurses menuconfig, xconfig +HERMETIC_TOOLCHAIN=0 + +set -e +set -a +source "${ROOT_DIR}/build/_setup_env.sh" +set -a + +# Disable mixed build +GKI_BUILD_CONFIG= + +function sort_config() { + # Normal sort won't work because all the "# CONFIG_.. is not set" would come + # before all the "CONFIG_..=m". Use sed to extract the CONFIG_ option and prefix + # the line in front of the line to create a key (e.g. CONFIG_.. # CONFIG_.. is not set), + # sort, then remove the key + sed -E -e 's/.*(CONFIG_[^ =]+).*/\1 \0/' $1 | sort -k1 | cut -F2- +} + +function menuconfig() { + set +x + local orig_config=$(mktemp) + local new_config="${OUT_DIR}/.config" + local changed_config=$(mktemp) + local new_fragment=$(mktemp) + + trap "rm -f ${orig_config} ${changed_config} ${new_fragment}" EXIT + + if [ -n "${FRAGMENT_CONFIG}" ]; then + if [[ -f "${ROOT_DIR}/${FRAGMENT_CONFIG}" ]]; then + FRAGMENT_CONFIG="${ROOT_DIR}/${FRAGMENT_CONFIG}" + elif [[ "${FRAGMENT_CONFIG}" != /* ]]; then + echo "FRAGMENT_CONFIG must be an absolute path or relative to ${ROOT_DIR}: ${FRAGMENT_CONFIG}" + exit 1 + elif [[ ! -f "${FRAGMENT_CONFIG}" ]]; then + echo "Failed to find FRAGMENT_CONFIG: ${FRAGMENT_CONFIG}" + exit 1 + fi + fi + + cp ${OUT_DIR}/.config ${orig_config} + (cd ${KERNEL_DIR} && make "${TOOL_ARGS[@]}" O=${OUT_DIR} ${MAKE_ARGS} ${1:-menuconfig}) + + if [ -z "${FRAGMENT_CONFIG}" ]; then + (cd ${KERNEL_DIR} && make "${TOOL_ARGS[@]}" O=${OUT_DIR} ${MAKE_ARGS} savedefconfig) + [ "$ARCH" = "x86_64" -o "$ARCH" = "i386" ] && local ARCH=x86 + echo "Updating ${ROOT_DIR}/${KERNEL_DIR}/arch/${ARCH}/configs/${DEFCONFIG}" + mv ${OUT_DIR}/defconfig ${ROOT_DIR}/${KERNEL_DIR}/arch/${ARCH}/configs/${DEFCONFIG} + return + fi + + ${KERNEL_DIR}/scripts/diffconfig -m ${orig_config} ${new_config} > ${changed_config} + KCONFIG_CONFIG=${new_fragment} ${ROOT_DIR}/${KERNEL_DIR}/scripts/kconfig/merge_config.sh -m ${FRAGMENT_CONFIG} ${changed_config} + sort_config ${new_fragment} > ${FRAGMENT_CONFIG} + set +x + + + echo + echo "Updated ${FRAGMENT_CONFIG}" + echo +} + +menucommand="${1:-menuconfig}" +MAKE_ARGS="${@:2}" + +if [[ "${menucommand}" =~ "*config" ]]; then + MAKE_ARGS="$*" + menucommand="menuconfig" +fi + +# let all the POST_DEFCONFIG_CMDS run since they may clean up loose files, then exit +append_cmd POST_DEFCONFIG_CMDS "exit" +# menuconfig should go first. If POST_DEFCONFIG_CMDS modifies the .config, then we probably don't +# want those changes to end up in the resulting saved defconfig +POST_DEFCONFIG_CMDS="menuconfig ${menucommand} && ${POST_DEFCONFIG_CMDS}" + +${ROOT_DIR}/build/build.sh ${MAKE_ARGS} diff --git a/copy_symbols.sh b/copy_symbols.sh new file mode 100755 index 0000000000..f64ecc0fff --- /dev/null +++ b/copy_symbols.sh @@ -0,0 +1,42 @@ +#!/bin/bash -eu + +# Copyright (C) 2021 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an internal helper script used by build.sh and build_abi.sh + +# arguments are: +# abi_sl - the dist directory ABI symbol list +# kernel_dir +# main_symbol_list_file_name +# (additional_symbol_list_file_name)* + +abi_sl="$1"; shift +kernel_dir="$1"; shift +symbol_list="$1"; shift + +# Copy the abi symbol list file from the sources into the dist dir. +verb=Generating +test -e "$abi_sl" && verb=Refreshing +echo "========================================================" +echo " $verb abi symbol list definition in $abi_sl" +cp -- "$kernel_dir/$symbol_list" "$abi_sl" + +# If there are additional symbol lists specified, append them. +for symbol_list; do + echo >> "$abi_sl" + cat -- "$kernel_dir/$symbol_list" >> "$abi_sl" +done + +exit 0 diff --git a/envsetup.sh b/envsetup.sh new file mode 100644 index 0000000000..09ab724669 --- /dev/null +++ b/envsetup.sh @@ -0,0 +1,43 @@ +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an implementation detail of build.sh and friends. Do not source +# directly as it will spoil your shell and make build.sh unusable. You have +# been warned! If you have a good reason to source the result of this file into +# a shell, please let kernel-team@android.com know and we are happy to help +# with your use case. + + +# This is a dummy to not break people that have a workflow that includes +# sourcing build/envsetup.sh into a shell when working with Android repo. +# The actual functionality of this script has been moved to _setup_env.sh. +# +# It turns out that build/envsetup.sh was sourced into the shell by a lot of +# people. Mostly due to the fact that old documentation asked people to do so +# (including this script itself). Unfortunately, this causes more harm than it +# does any good. Mostly it spoils the shell with environment variables that are +# only valid in the context of a very specific build configuration. To overcome +# this, the content of this file has been moved to _setup_env.sh and callers +# within this project have been adjusted. This script serves as a dummy to not +# break people sourcing it, but it will from now on emit a deprecation warning. +# That script might be removed at a later time. +# +# For further information on the Android Kernel build process with the tooling +# of this project, please refer to +# https://source.android.com/setup/build/building-kernels. +# +# For any questions or concerns, please contact kernel-team@android.com. + +echo "Sourcing 'build/envsetup.sh' for Android Kernels is deprecated and no longer valid!" +echo "Please refer to the documentation in said script for details." diff --git a/gki/README.md b/gki/README.md new file mode 100644 index 0000000000..66d269430d --- /dev/null +++ b/gki/README.md @@ -0,0 +1,136 @@ +GKI Tools for Android Kernels +============================= + +Overview +-------- + +This directory contains helpful tools that may be used to aid in the +development of modularized drivers. + +add_EXPORT_SYMBOLS_GPL +---------------------- + + USAGE: add_EXPORT_SYMBOL_GPL [--no-skip-arch] < kernel_build_error_log + add_EXPORT_SYMBOL_GPL [--no-skip-arch] kernel_build_error_log + grep /[.]ko build_error_log | add_EXPORT_SYMBOL_GPL [--no-skip-arch] + vi `add_EXPORT_SYMBOL_GPL [--no-skip-arch] < kernel_build_error_log` + +To acquire the kernel_build_error_log eg: + + $ ./build_sm8250.sh -j50 2>&1 | tee kernel_build_error_log + +To only create commit related to symbols needed for cam_spec.ko module: + + $ grep /cam_spec[.]ko kernel_build_error_log | add_EXPORT_SYMBOL_GPL + +To only create commit related to a specific list of symbols, there is +the option to land just the symbols, no spaces, one per line, into a +manufactured or edited kernel_build_error_log and feed that to the script. + +The script will only affect the current directory level and downward, +this allows one to segregate the adjusted content. Any symbols that +are needed outside the range of that directory will result in errors +and the git commit phase will not be performed. + +Add EXPORT_SYMBOL_GPL for any noted missing symbols, output the list of files +modified to stdout (so it can be passed to an editor command line should you +need to check or adjust the results). Automatically commit the list of files +into git. + +Deals as simply as it can to handle \_\_trace\_\, sorting the result. + +Keep in mind exports can change, be added or subtracted, and that preliminary +work may expose or remove required symbols to resolve during later work. As +such this script only adds, so you may need to revert the results and try +again to get the most up to date set. By making this part automated it can +deal with the tens or thousands of exports that need to be discovered or +added. If you need to adjust a subsystem, run this script in the subsystem +directory, and it will only adjust from that point downwards leaving other +higher up trees alone. + +add_MODULE_LICENSE +------------------ + + USAGE: add_MODULE_LICENSE < kernel_build_error_log + add_MODULE_LICENSE kernel_build_error_log + +Add MODULE_LICENSE to all the files. + +Must be performed in the root directory. + +find_circular +------------- + + USAGE: find_circular [dir] + +Call this when depmod breaks down, or when one needs a list of the symbols +implicated in the circular dependency. + +Search current or dir directory for all kernel modules. Itemize what they +export, and what they import. Discover links and report who fulfills them. +Report any first order circular relationships and the symbols that got us +into the situation. + +Standard output is of the form: + +module1.ko(symbols) -> module2.ko(symbols) -> module1.ko + +Leaves an annotated modules.dep file in the specified directory. + +device_snapshot +--------------- + + USAGE: device_snapshot [-s ] [-D] [-f []] [-F [-o [-d ]]] + +Collect filtered /dev and /sys details, along with dmesg and probe list. + +-o \ will drop the collection into a set of files, but will not +overrite existing content. -F will overwrite. + +-D will wait for the display + +if \ is empty ('' or last option), will not collect dmesg or probe +list. If no -o option is specified, then \ will be default of - +(stdout) and all pieces will go to the standard output separated by a cut +and snip header. If specified, \ will contain the filtered /dev/ +and /sys/ dumps, \.probed the subset filter of just the probed drivers, +\.dmesg the kernel logs and \.config the uncompressed +/proc/config.gz. + +-d \ will take the dropped collection specified to -o \ and +produce a diff -U1 output compared against the \. + +-f \ allows one to utilize the filter to an existing find /dev /sys +output from a device. No dmesg will be collected. + +-s \ will allow one to specify a device to connect to when multiples +are available, otherwise will default to one available or ANDROID_SERIAL +environment variable. + +In your local build/flash/boot script for tight development cycles, add + + SEQ=`for i in out/${DEFAULT_BUILD}.snapshot.[0-9]*; do + echo ${i#out/${DEFAULT_BUILD}.snapshot.} + done | + sed 's/^0*//' | + grep -v 0-9 | + tr -d .[:alpha:] | + sort -nu | + tail -1` && + NEWSEQ=$((${SEQ:-0}+1)) && + NEWSEQ=`printf "%03u" ${NEWSEQ}` + if [ -z "${SEQ}" ]; then + private/msm-google/scripts/gki/device_snapshot \ + -o out/${DEFAULT_BUILD}.snapshot.${NEWSEQ} + else + SEQ=`printf "%03u" ${SEQ}` + private/msm-google/scripts/gki/device_snapshot \ + -o out/${DEFAULT_BUILD}.snapshot.${NEWSEQ} \ + -d out/${DEFAULT_BUILD}.snapshot.${SEQ} + fi + +instrument_module_init +---------------------- + USAGE: instrument_module_init [dir|file] + +Add debug instrumentation to module_init and probe functions. diff --git a/gki/add_EXPORT_SYMBOL_GPL b/gki/add_EXPORT_SYMBOL_GPL new file mode 100755 index 0000000000..47845d9adc --- /dev/null +++ b/gki/add_EXPORT_SYMBOL_GPL @@ -0,0 +1,398 @@ +#! /bin/sh +# SPDX-License-Identifier: Apache-2.0 +# +# (c) 2019, Google +progname="${0##*/}" + +# Change to EXPORT_SYMBOL, EXPORT_SYMBOL_GPL or other namespace variant default +EXPORT_SYMBOL=${EXPORT_SYMBOL_GPL:-${progname#add_}} + +USAGE="USAGE: ${progname} [--no-skip-arch] < kernel_build_error_log + ${progname} [--no-skip-arch] kernel_build_error_log + grep /[.]ko build_error_log | ${progname} [--no-skip-arch] + ${EDITOR:-vi} \`${progname} [--no-skip-arch] < kernel_build_error_log\` + +To acquire the kernel_build_error_log eg: +\$ ./build_sm8250.sh -j50 2>&1 | tee kernel_build_error_log + +To only create commit related to symbols needed for cam_spec.ko module: +\$ grep /cam_spec[.]ko kernel_build_error_log | ${progname} + +To only create commit related to a specific list of symbols, there is +the option to land just the symbols, no spaces, one per line, into a +manufactured or edited kernel_build_error_log and feed that to the script. + +The script will only affect the current directory level and downward, +this allows one to segregate the adjusted content. Any symbols that +are needed outside the range of that directory will result in errors +and the git commit phase will not be performed. + +Add ${EXPORT_SYMBOL} for any noted missing symbols, output the list of files +modified to stdout (so it can be passed to an editor command line should you +need to check or adjust the results). Automatically commit the list of files +into git. + +Deals as simply as it can to handle __trace_, sorting the result. + +Keep in mind exports can change, be added or subtracted, and that preliminary +work may expose or remove required symbols to resolve during later work. As +such this script only adds, so you may need to revert the results and try +again to get the most up to date set. By making this part automated it can +deal with the tens or thousands of exports that need to be discovered or +added. If you need to adjust a subsystem, run this script in the subsystem +directory, and it will only adjust from that point downwards leaving other +higher up trees alone." + +if [ X"--help" = X"${1}" -o X"{-h}" = X"${1}" ]; then + echo "${USAGE}" >&2 + exit +fi +skip_arch=true +if [ X"--no-skip-arch" = X"${1}" ]; then + skip_arch=false + shift +fi +INPUT= +if [ 1 = ${#} ]; then + INPUT=${1} + shift +fi +if [ 0 != ${#} ]; then + echo "Unexpected Argument: ${*}" >&2 + echo >&2 + echo "${USAGE}" >&2 + exit 1 +fi + +# A _real_ embedded tab character +TAB="`echo | tr '\n' '\t'`" +# A _real_ embedded escape character +ESCAPE="`echo | tr '\n' '\033'`" +# A _real_ embedded carriage return character +CR="`echo | tr '\n' '\r'`" +# Colours +RED="${ESCAPE}[38;5;196m" +BLUE="${ESCAPE}[35m" +NORMAL="${ESCAPE}[0m" + +# Common grep/sed regex expressions +STRUCT_TYPE="struct[ ${TAB}][^ ${TAB}]\{1,\}" +VAR_TYPE="\(${STRUCT_TYPE}\|bool\|char\|int\|long\|long long\|u\{0,1\}int[0-9]*t\)[ ${TAB}]\{1,\}[*]\{0,1\}" +STRUCT_TYPE="${STRUCT_TYPE}[ ${TAB}]\{1,\}[*]\{0,1\}" + +# Files that exports should never exist, or need to be carefully considered +skip_arch_files() { + if ${skip_arch}; then + grep -v '\(^\|^[^:]*/\)arch/[^a][^r][^m][^:]*:' + else + cat - + fi +} + +# Following sucks in all the data from stdin or ${INPUT} +# Check for depmod output (undefined!) or built-in kernel complaints (undefined +# references). The later is not advised because it means the kernel may be +# dependent on a module, we collect it to provide a signal for future work. +echo "Reading symbols" >&2 +MODULES="` + cat ${INPUT:--} | + tr -d \"${CR}\" | + sed -n \ + -e '/^abi_gki_.*_whitelist$/d' \ + -e 's/^[_a-zA-Z][_a-zA-Z0-9]\{6,\}$/&/p' \ + -e 's/^ERROR: "\([^"]*\)" [[]\(.*[.]ko\)[]] undefined!$/\1 \2/p' \ + -e 's/^[^:]*:[0-9]*: undefined reference to \`\([^'\'']*\)'\''$/\1/p' | + sort -u`" +SYMBOLS="`echo \"${MODULES}\" | sed 's/ .*$//' | sort -u`" +if [ -z "${SYMBOLS}" ]; then + echo "${BLUE}WARNING${NORMAL}: no symbols found" >&2 + exit +fi + +SYMBOLS_PLUS_TRACE="${SYMBOLS}" +TRACE_MODULES="`echo \"${MODULES}\" | + sed -n 's/__tracepoint_.* //p' | + sort -u`" +TRACE_MODULES_2= +c=`echo "${TRACE_MODULES}" | wc -l` +if [ 1 -eq ${c} ];then + TRACE_MODULES="${TRACE_MODULES%/*.ko}" + if [ -z "${INPUT}" ]; then + TRACE_MODULES_2=* + else + TRACE_MODULES_2=${INPUT} + fi +elif [ -z "${INPUT}" ]; then + TRACE_MODULES=* +else + TRACE_MODULES=${INPUT} +fi +TRACE_FILE="`grep -rl '^#define CREATE_TRACE_POINTS' ${TRACE_MODULES}`" +if [ -z "${TRACE_FILE}" -a -n "${TRACE_MODULES_2}" ]; then + TRACE_FILE="`grep -rl '^#define CREATE_TRACE_POINTS' ${TRACE_MODULES_2}`" +fi +c=0 +if [ -n "${TRACE_FILE}" ]; then + c=`echo "${TRACE_FILE}" | wc -l` +fi +if [ 1 -eq ${c} ]; then + SYMBOLS_PLUS_TRACE="${SYMBOLS} +`echo \"${SYMBOLS}\" | + sed -n 's/^__tracepoint_\(.*\)/EXPORT_TRACEPOINT_SYMBOL_GPL(\1);/p'`" +else + TRACE_FILE= +fi + +echo "Finding symbols in the current tree" >&2 +DATA="`grep -Fr \"${SYMBOLS_PLUS_TRACE}\" * | + grep \"^[^:]*[.][cS]:[^ ${TAB}]\" | + skip_arch_files`" +echo "Editing the files that contain the symbols" >&2 +TMP=`mktemp -d` + +[ 'USAGE: report_strip_of_tag tag + +Report if a problematic function segment tag is stripped from the file + +expects ${F}, ${TMP}, ${TAB}, ${RED} and ${NORMAL} to be defined' ] +report_strip_of_tag() { + if diff --side-by-side --suppress-common-lines ${F} ${TMP}/${F##*/} | + sed -n "s/\(.*[ ${TAB}]\)\(__${1}\)\([^a-zA-Z0-9_].*[^ ${TAB}]\)[ ${TAB}]\{1,\}|[ ${TAB}]\{1,\}/\1${BLUE}\2${NORMAL}\3 -> /p" | + grep __${1} >&2; then + echo "${RED}ERROR${NORMAL}: scrubbing __${1} from function in ${F}" >&2 + echo " Compile check, possibly edit ${F} and friends." >&2 + echo " Then re-run ${progname} to complete." >&2 + echo FAILED + fi +} + +for s in ${SYMBOLS}; do + # Already there? + if echo "${DATA}" | + grep "EXPORT_\(TRACEPOINT_SYMBOL_GPL*(${s#__tracepoint_}\|SYMBOL\(_GPL\)*(${s}\))" >/dev/null; then + echo INFO: ${s} found and already exported >&2 + continue + fi + m="`echo \"${DATA}\" | grep \"^[^:]*:\([^ ${TAB}].*[ *]\|\)${s}(\"`" + c=0 + if [ -n "${m}" ]; then + c=`echo "${m}" | wc -l` + fi + if [ 0 -eq ${c} ]; then + m="`echo \"${DATA}\" | grep \"^[^:]*:\(const[ ${TAB}]\{1,\}\|\)${STRUCT_TYPE}${s}\([[][]]\|\) = {\"`" + c=0 + if [ -n "${m}" ]; then + c=`echo "${m}" | wc -l` + fi + if [ 0 -eq ${c} ]; then + m="`echo \"${DATA}\" | grep \"^[^:]*:\(const[ ${TAB}]\{1,\}\|\)${VAR_TYPE}${s}[\[ ${TAB};=]\"`" + c=0 + if [ -n "${m}" ]; then + c=`echo "${m}" | wc -l` + fi + if [ 0 -eq ${c} ]; then + if [ -z "${TRACE_FILE}" -o "${s}" = "${s#__tracepoint_}" ]; then + echo "${RED}ERROR${NORMAL}: ${s} not found" >&2 + if [ "${s}" != "${s#__tracepoint_}" -o "${s}" = "${s#__}" ]; then + echo FAILED + else + echo "${RED}WARNING${NORMAL}: ${s} might be a linker variable? skipping marking as failure" >&2 + fi + else + echo ${TRACE_FILE} ${s} + fi + elif [ 1 -eq ${c} ]; then + echo "${DATA}" | + sed -n "s/^\([^:]*\):\(const[ $TAB]\{1,\}\|\)${VAR_TYPE}${s}.*/\1 ${s}/p" + else + echo "${RED}ERROR${NORMAL}: ${c} matches: `echo \"${m}\" | + tr '\n' '&' | + sed -e 's/&$//' \ + -e 's/&/ & /g'`" >&2 + echo FAILED + fi + elif [ 1 -eq ${c} ]; then + echo "${DATA}" | + sed -n "s/^\([^:]*\):\(const[ $TAB]\{1,\}\|\)${STRUCT_TYPE}\(${s}\)\([[][]]\|\) = {.*/\1 \3/p" + else + echo "${RED}ERROR${NORMAL}: ${c} matches: `echo \"${m}\" | + tr '\n' '&' | + sed -e 's/&$//' \ + -e 's/&/ & /g'`" >&2 + echo FAILED + fi + elif [ 1 -eq ${c} ]; then + echo "${DATA}" | + sed -n "s/^\([^:]*\):\([^ ${TAB}].*[ *]\|\)\(${s}\)(.*/\1 \3/p" + else + echo "${RED}ERROR${NORMAL}: ${c} matches: `echo \"${m}\" | + tr '\n' '&' | + sed -e 's/&$//' \ + -e 's/&/ & /g'`" >&2 + echo FAILED + fi +done | + sort -u | + while read F s; do + if [ "FAILED" = "${F}" ]; then + echo FAILED + continue + fi + if [ "${s}" != "${s#__tracepoint_}" ]; then + sed "\$ { + /^[^E]/ { + a \ + + } + a \ + newEXPORT_TRACEPOINT_SYMBOL_GPL(${s#__tracepoint_}); + }" ${F} + else + cat ${F} + fi | + sed "/^\([^ ${TAB}].*[ *]\|\)${s}(/ { + : loop1 + N + /\(\n}\);*$/ { + s//\1/ + s/^\([^\n]*[^\n ${TAB}]\|\)[ ${TAB}]\{1,\}__\(init\|exit\)[ ${TAB}]\{1,\}\([^\n]*(\)/\1 \3/ + s/^\([^\n]*[^\n ${TAB}]\|\)[ ${TAB}]\{1,\}__\(init\|exit\)\([^ ${TAB}a-zA-Z0-9_][^\n]*(\)/\1 \3/ + s/^__\(init\|exit\)[ ${TAB}]\{1,\}\([^\n]*(\)/\2/ + s/^__\(init\|exit\)\([^ ${TAB}a-zA-Z0-9_][^\n]*(\)/\2/ + a \ + new${EXPORT_SYMBOL}(${s}); + b next1 + } + b loop1 + : next1 + } + /^\(const[ ${TAB}]\{1,\}\|\)${STRUCT_TYPE}${s}\([[][]]\|\)[ ${TAB}]*=[ ${TAB}]*{/ { + : loop2 + N + /\n};$/ { + a \ + new${EXPORT_SYMBOL}(${s}); + b next2 + } + b loop2 + : next2 + } + /^\(const[ ${TAB}]\{1,\}\|\)${VAR_TYPE}${s}\(\([[][]]\|\)[ ${TAB}]*=.*\|\);/ { + a \ + new${EXPORT_SYMBOL}(${s}); + } + /^\(const[ ${TAB}]\{1,\}\|\)${VAR_TYPE}${s}\([[][]]\|\)[ ${TAB}]*=[ ${TAB}]*{\{0,1\}\$/ { + : loop3 + N + /;$/ { + a \ + new${EXPORT_SYMBOL}(${s}); + b next3 + } + b loop3 + : next3 + }" | + sed '/^newEXPORT/ { + s//EXPORT/ + N + : loop + N + s/\(\n\)\n$/\1/ + t loop + }' >${TMP}/${F##*/} && + ! cmp -s ${F} ${TMP}/${F##*/} && + [ -s ${TMP}/${F##*/} ] && + report_strip_of_tag init && + report_strip_of_tag exit && + cp ${TMP}/${F##*/} ${F} && + echo INFO: export ${s} in ${F} >&2 || + ( + echo "${RED}ERROR${NORMAL}: export ${s} in ${F}" >&2 + echo FAILED + ) + echo ${F} + rm ${TMP}/${F##*/} + done >${TMP}/${progname}.out +FILES="`grep -v '^FAILED$' ${TMP}/${progname}.out | sort -u`" +echo "${FILES}" +grep '^FAILED$' ${TMP}/${progname}.out >/dev/null +ret=${?} +rm -rf ${TMP} +if [ 0 -ne ${ret} ]; then + echo "DONE" >&2 + GIT_FILES="`git diff | + sed -n 's@^diff --git a/\(.*\) b/\1$@\1@p' | + sort -u`" + if [ -z "${GIT_FILES}" ]; then + echo "${BLUE}WARNING${NORMAL}: no changes to commit" >&2 + if [ -n "${FILES}" ]; then + echo " but we altered files!" >&2 + fi + exit + fi + SEMI="" + if [ X"${GIT_FILES}" != X"`echo ${FILES} | tr ' ' '\n' | sort -u`" ]; then + SEMI="Semi-" + FILES=`echo ${FILES} ${GIT_FILES} | + tr ' ' '\n' | + sort -u` + echo "${BLUE}WARNING${NORMAL}: list of files in git different from files we expected to edit, continuation?" >&2 + fi + + AUTHOR_NAME="`git config user.name`" + AUTHOR_EMAIL="`git config user.email`" + if [ -z "${AUTHOR_EMAIL}" ]; then + AUTHOR_EMAIL="`${USER}@google.com`" + fi + if [ -z "${AUTHOR_NAME}" ]; then + convert_ldap_to_name() { + echo "${1##*/}" | + sed "s@[-_]@ @g + s@ *@ @g" | + sed 's@\(^\| \)a@\1A@g + s@\(^\| \)b@\1B@g + s@\(^\| \)c@\1C@g + s@\(^\| \)d@\1D@g + s@\(^\| \)e@\1E@g + s@\(^\| \)f@\1F@g + s@\(^\| \)g@\1G@g + s@\(^\| \)h@\1H@g + s@\(^\| \)i@\1I@g + s@\(^\| \)j@\1J@g + s@\(^\| \)k@\1K@g + s@\(^\| \)l@\1L@g + s@\(^\| \)m@\1M@g + s@\(^\| \)n@\1N@g + s@\(^\| \)o@\1O@g + s@\(^\| \)p@\1P@g + s@\(^\| \)q@\1Q@g + s@\(^\| \)r@\1R@g + s@\(^\| \)s@\1S@g + s@\(^\| \)t@\1T@g + s@\(^\| \)u@\1U@g + s@\(^\| \|Mc\)v@\1V@g + s@\(^\| \)w@\1W@g + s@\(^\| \)x@\1X@g + s@\(^\| \)y@\1Y@g + s@\(^\| \)z@\1Z@g' + } + AUTHOR_NAME="`convert_ldap_to_name ${USER}`" + fi + + git commit ${FILES} -m "GKI: add missing exports for =m for + + + +${SEMI}Automatically generated by ${0##*/} + +The following symbols are exported: +`echo \"${SYMBOLS}\" | + sed 's/^/ - /'` + +Signed-off-by: ${AUTHOR_NAME} <${AUTHOR_EMAIL}> +Test: compile" || + echo "${RED}ERROR${NORMAL}: failed to commit changes to git" >&2 + exit +fi +echo "${RED}FAILED${NORMAL}: could not complete task" >&2 +exit 1 diff --git a/gki/add_MODULE_LICENSE b/gki/add_MODULE_LICENSE new file mode 100755 index 0000000000..235690c5a6 --- /dev/null +++ b/gki/add_MODULE_LICENSE @@ -0,0 +1,112 @@ +#! /bin/sh +# SPDX-License-Identifier: Apache-2.0 +# +# (c) 2019, Google +progname="${0##*/}" + +USAGE="USAGE: ${progname} < kernel_build_error_log + ${progname} kernel_build_error_log + +Add MODULE_LICENSE to all the files. + +Must be performed in the root directory." + +if [ X"--help" = X"${1}" -o X"{-h}" = X"${1}" ]; then + echo "${USAGE}" >&2 + exit +fi +INPUT= +if [ 1 = ${#} ]; then + INPUT=${1} + shift +fi +if [ 0 != ${#} ]; then + echo "Unexpected Argument: ${*}" >&2 + echo >&2 + echo "${USAGE}" >&2 + exit 1 +fi + +convert_to_module_name() { + echo "${1##*/}" | + sed 's@[-_]@ @g + s@ *@ @g + s@^btfm @BTFM @ + s@^msm @MSM @ + s@ dma\( \|$\)@ DMA\1@g + s@ irq\( \|$\)@ IRQ\1@g + s@ io\( \|$\)@ I/O\1@g + s@ mem\( \|$\)@ memory\1@g + s@ hw\( \|$\)@ hardware\1@g + s@ jpeg\( \|$\)@ JPEG\1@g + s@ mgr$@ manager@ + s@ util$@ utilities@ + s@\(^\| \)a@\1A@g + s@\(^\| \)b@\1B@g + s@\(^\| \)c@\1C@g + s@\(^\| \)d@\1D@g + s@\(^\| \)e@\1E@g + s@\(^\| \)f@\1F@g + s@\(^\| \)g@\1G@g + s@\(^\| \)h@\1H@g + s@\(^\| \)i@\1I@g + s@\(^\| \)j@\1J@g + s@\(^\| \)k@\1K@g + s@\(^\| \)l@\1L@g + s@\(^\| \)m@\1M@g + s@\(^\| \)n@\1N@g + s@\(^\| \)o@\1O@g + s@\(^\| \)p@\1P@g + s@\(^\| \)q@\1Q@g + s@\(^\| \)r@\1R@g + s@\(^\| \)s@\1S@g + s@\(^\| \)t@\1T@g + s@\(^\| \)u@\1U@g + s@\(^\| \)v@\1V@g + s@\(^\| \)w@\1W@g + s@\(^\| \)x@\1X@g + s@\(^\| \)y@\1Y@g + s@\(^\| \)z@\1Z@g' +} + +sed -n \ + 's/WARNING: modpost: missing MODULE_LICENSE() in \(.*\)[.]o$/\1/p' \ + ${INPUT} | + sort -u | + while read f; do + F=${f}.c + if [ ! -f "${F}" ]; then + F=${f%${f##*/}}`echo ${f##*/} | tr -- - _`.c + if [ ! -f "${F}" ]; then + F=${F%.c}_core.c + if [ ! -f "${F}" ]; then + F=${f%${f##*/}}`echo ${f##*/} | tr _ -`.c + if [ ! -f "${F}" ]; then + F=${F%.c}_core.c + if [ ! -f "${F}" ]; then + echo WARNING: ${f}.c not found, needs a MODULE_LICENSE >&2 + continue + fi + fi + fi + fi + fi + if grep MODULE_LICENSE ${F} >/dev/null 2>/dev/null; then + echo INFO: MODULE_LICENSE already present in ${F} >&2 + continue + fi + module="`convert_to_module_name ${f}`" + if [ -z "${module}" ]; then + echo ERROR: ${f} can not be converted into a module name >&2 + continue + fi + sed -i "\${ + a \\ +\\ +MODULE_LICENSE(\"GPL v2\");\\ +MODULE_DESCRIPTION(\"${module}\"); + }" ${F} && + echo INFO: Added MODULE_LICENSE to ${F} >&2 || + echo ERROR: Failed to add MODULE_LICENSE to ${F} >&2 + done + diff --git a/gki/device_snapshot b/gki/device_snapshot new file mode 100755 index 0000000000..01226dc1bb --- /dev/null +++ b/gki/device_snapshot @@ -0,0 +1,778 @@ +#! /bin/sh +# SPDX-License-Identifier: Apache-2.0 +# +# (c) 2019, Google + +progname=${0##*/} +NUM=XX +USAGE="USAGE: ${progname} [-s ] [-D] [-f []] [-F [-o [-d ]]] + +Collect filtered /dev and /sys details, along with dmesg and probe list. + +-o will drop the collection into a set of files, but will not +overwrite existing content. -F will overwrite. + +-D will wait for the display. + +if is empty ('' or last option), will not collect dmesg or probe +list. If no -o option is specified, then will be default of - +(stdout) and all pieces will go to the standard output separated by a cut +and snip header. If specified, will contain the filtered /dev/ +and /sys/ dumps, .probed the subset filter of just the probed drivers, +.dmesg the kernel logs and .config the uncompressed +/proc/config.gz. + +-d will take the dropped collection specified to -o and +produce a diff -U1 output compared against the . + +-f allows one to utilize the filter to an existing find /dev /sys +output from a device. No dmesg will be collected. + +-s will allow one to specify a device to connect to when multiples +are available, otherwise will default to one available or ANDROID_SERIAL +environment variable. + +NB: The /dev/ and /sys/ dumps can be compared with something like a diff -U1 + and report a generally noiseless result so it is easy to spot the side + effects from one build to the next. Basically nothing else should be + added or removed from the differences other than the expectations, save + for normal noise from the dynamic nature of the kernel, thus making it + beneficial to run this script collection at the same point in the boot + process to keep noise down. + +NB: for default standard output, the probed list will be post-filter so + some details may be altered to make it easier to discover differences. + So if bus enumerations change, some of the enumerators will be replaced + with an '${NUM}' so that they continue to match when moved. For a + specified output, the .probed file will retain the details." + +EMPTY="" +SPACE=" " +# A _real_ embedded tab character +TAB="`echo | tr '\n' '\t'`" +# A _real_ embedded carriage return character +CR="`echo | tr '\n' '\r'`" +# A _real_ embedded escape character +ESCAPE="`echo | tr '\n' '\033'`" +# Colours +RED="${ESCAPE}[38;5;196m" +ORANGE="${ESCAPE}[38;5;255:165:0m" +BLUE="${ESCAPE}[35m" +NORMAL="${ESCAPE}[0m" + +## +## Helper Functions +## + +[ "USAGE: inFastboot + +Returns: true if device is in fastboot mode" ] +inFastboot() { + fastboot devices | + if [ -n "${ANDROID_SERIAL}" ]; then + grep "^${ANDROID_SERIAL}[${SPACE}${TAB}]" > /dev/null + else + wc -l | grep '^1$' >/dev/null + fi +} + +[ "USAGE: inAdb + +Returns: true if device is in adb mode" ] +inAdb() { + adb devices /dev/null + else + wc -l | grep '^1$' >/dev/null + fi +} + +[ "USAGE: inRecovery + +Returns: true if device is in recovery mode" ] +inRecovery() { + local list="`adb devices | + grep -v -e 'List of devices attached' -e '^$'`" + if [ -n "${ANDROID_SERIAL}" ]; then + echo "${list}" | + grep "^${ANDROID_SERIAL}[${SPACE}${TAB}][${SPACE}${TAB}]*recovery\$" >/dev/null + return ${?} + fi + if echo "${list}" | wc -l | grep '^1$' >/dev/null; then + echo "${list}" | + grep "[${SPACE}${TAB}]recovery\$" >/dev/null + return ${?} + fi + false +} + +[ "USAGE: USB_DEVICE=\`usb_devnum [--next]\` + +USB_DEVICE contains cache. Update if system changes. + +Returns: the devnum for the USB_SERIAL device" ] +usb_devnum() { + if [ -n "${USB_SERIAL}" ]; then + local usb_device=`cat ${USB_SERIAL%/serial}/devnum 2>/dev/null | tr -d ' \t\r\n'` + if [ -n "${usb_device}" ]; then + USB_DEVICE=dev${usb_device} + elif [ -n "${USB_DEVICE}" -a "${1}" ]; then + USB_DEVICE=dev`expr ${USB_DEVICE#dev} + 1` + fi + echo "${USB_DEVICE}" + fi +} + +[ "USAGE: adb_sh /dev/stdout 2>/dev/stderr + +Returns: true if the command succeeded" ] +adb_sh() { + local args= + for i in "${@}"; do + [ -z "${args}" ] || args="${args} " + if [ X"${i}" != X"${i#\'}" ]; then + args="${args}${i}" + elif [ X"${i}" != X"${i#*\\}" ]; then + args="${args}`echo ${i} | sed 's/\\\\/\\\\\\\\/g'`" + elif [ X"${i}" != X"${i#* }" ]; then + args="${args}'${i}'" + elif [ X"${i}" != X"${i#*${TAB}}" ]; then + args="${args}'${i}'" + else + args="${args}${i}" + fi + done + adb shell "${args}" +} + +[ "USAGE: adb_su /dev/stdout 2>/dev/stderr + +Returns: true if the command running as root succeeded" ] +adb_su() { + adb_sh su root "${@}" +} + +[ "USAGE: get_property + +Returns the property value" ] +get_property() { + adb_sh getprop ${1} 2>/dev/null |s|m|h|d] + +human readable output whole seconds, whole minutes or mm:ss" ] +format_duration() { + if [ -z "${1}" ]; then + echo unknown + return + fi + local duration="${1}" + if [ X"${duration}" != X"${duration%s}" ]; then + duration=${duration%s} + elif [ X"${duration}" != X"${duration%m}" ]; then + duration=`expr ${duration%m} \* 60` + elif [ X"${duration}" != X"${duration%h}" ]; then + duration=`expr ${duration%h} \* 3600` + elif [ X"${duration}" != X"${duration%d}" ]; then + duration=`expr ${duration%d} \* 86400` + fi + local seconds=`expr ${duration} % 60` + local minutes=`expr \( ${duration} / 60 \) % 60` + local hours=`expr ${duration} / 3600` + if [ 0 -eq ${minutes} -a 0 -eq ${hours} ]; then + if [ 1 -eq ${duration} ]; then + echo 1 second + return + fi + echo ${duration} seconds + return + elif [ 60 -eq ${duration} ]; then + echo 1 minute + return + elif [ 0 -eq ${seconds} -a 0 -eq ${hours} ]; then + echo ${minutes} minutes + return + fi + if [ 0 -eq ${hours} ]; then + echo ${minutes}:`expr ${seconds} / 10``expr ${seconds} % 10` + return + fi + echo ${hours}:`expr ${minutes} / 10``expr ${minutes} % 10`:`expr ${seconds} / 10``expr ${seconds} % 10` +} + +[ "USAGE: adb_wait [timeout] + +Returns: waits until the device has returned for adb or optional timeout" ] +adb_wait() { + local start=`date +%s` + local duration= + local ret + if [ -n "${1}" ]; then + USB_DEVICE=`usb_devnum --next` + duration=`format_duration ${1}` + echo -n ". . . waiting ${duration}" ${ANDROID_SERIAL} ${USB_ADDRESS} ${USB_DEVICE} "${CR}" + timeout --preserve-status --signal=KILL ${1} adb wait-for-device 2>/dev/null + ret=${?} + echo -n " ${CR}" + else + adb wait-for-device + ret=${?} + fi + USB_DEVICE=`usb_devnum` + if [ 0 = ${ret} -a -n "${ACTIVE_SLOT}" ]; then + local active_slot=`get_active_slot` + if [ X"${ACTIVE_SLOT}" != X"${active_slot}" ]; then + echo "${ORANGE}[ WARNING ]${NORMAL} Active slot changed from ${ACTIVE_SLOT} to ${active_slot}" >&2 + fi + fi + local end=`date +%s` + local diff_time=`expr ${end} - ${start}` + local _print_time=${print_time} + if [ ${diff_time} -lt 15 ]; then + _print_time=false + fi + diff_time=`format_duration ${diff_time}` + if [ "${diff_time}" = "${duration}" ]; then + _print_time=false + fi + + local reason= + if inAdb; then + reason=`get_property ro.boot.bootreason` + fi + case ${reason} in + reboot*) + reason= + ;; + ${EMPTY}) + ;; + *) + reason=" for boot reason ${reason}" + ;; + esac + if ${_print_time} || [ -n "${reason}" ]; then + echo "${BLUE}[ INFO ]${NORMAL} adb wait duration ${diff_time}${reason}" + fi >&2 + + return ${ret} +} + +[ "USAGE: fastboot_wait [timeout] + +Returns: waits until the device has returned for fastboot or optional timeout" ] +fastboot_wait() { + local ret + # fastboot has no wait-for-device, but it does an automatic + # wait and requires (even a nonsensical) command to do so. + if [ -n "${1}" ]; then + USB_DEVICE=`usb_devnum --next` + echo -n ". . . waiting `format_duration ${1}`" ${ANDROID_SERIAL} ${USB_ADDRESS} ${USB_DEVICE} "${CR}" + timeout --preserve-status --signal=KILL ${1} fastboot wait-for-device >/dev/null 2>/dev/null + ret=${?} + echo -n " ${CR}" + ( exit ${ret} ) + else + fastboot wait-for-device >/dev/null 2>/dev/null + fi || + inFastboot + ret=${?} + USB_DEVICE=`usb_devnum` + if [ 0 = ${ret} -a -n "${ACTIVE_SLOT}" ]; then + local active_slot=`get_active_slot` + if [ X"${ACTIVE_SLOT}" != X"${active_slot}" ]; then + echo "${ORANGE}[ WARNING ]${NORMAL} Active slot changed from ${ACTIVE_SLOT} to ${active_slot}" + fi >&2 + fi + return ${ret} +} + +[ "USAGE: recovery_wait [timeout] + +Returns: waits until the device has returned for recovery or optional timeout" ] +recovery_wait() { + local ret + if [ -n "${1}" ]; then + USB_DEVICE=`usb_devnum --next` + echo -n ". . . waiting `format_duration ${1}`" ${ANDROID_SERIAL} ${USB_ADDRESS} ${USB_DEVICE} "${CR}" + timeout --preserve-status --signal=KILL ${1} adb wait-for-recovery 2>/dev/null + ret=${?} + echo -n " ${CR}" + else + adb wait-for-recovery + ret=${?} + fi + USB_DEVICE=`usb_devnum` + if [ 0 = ${ret} -a -n "${ACTIVE_SLOT}" ]; then + local active_slot=`get_active_slot` + if [ X"${ACTIVE_SLOT}" != X"${active_slot}" ]; then + echo "${ORANGE}[ WARNING ]${NORMAL} Active slot changed from ${ACTIVE_SLOT} to ${active_slot}" + fi >&2 + fi + return ${ret} +} + +[ "any_wait [timeout] + +Returns: waits until a device has returned or optional timeout" ] +any_wait() { + ( + adb_wait ${1} & + adb_pid=${!} + fastboot_wait ${1} & + fastboot_pid=${!} + recovery_wait ${1} & + recovery_pid=${!} + wait -n + kill "${adb_pid}" "${fastboot_pid}" "${recovery_pid}" + ) >/dev/null 2>/dev/null + inFastboot || inAdb || inRecovery +} + +## +## RUNTIME +## + +filter= +output=- +input= +display=false +force=false +while [ ${#} -gt 0 ]; do + case ${1} in + -f=* | --filter=*) + filter="${1#-f=}" + filter="${filter#--filter=}" + ;; + -f | --filter) + if [ ${#} = 1 -o -z "${2}" ]; then + filter=- + else + shift + filter="${1}" + fi + ;; + -F | --force) + force=true + ;; + -h | -\? | --help) + echo "${USAGE}" + exit 0 + ;; + -o=* | --output=*) + output="${1#-o=}" + output="${output#--output=}" + ;; + -o | --output) + if [ ${#} = 1 -o -z "${2}" ]; then + # empty output results in no dmesg or probed collection + output= + else + shift + output="${1}" + fi + ;; + -d=* | --diff=*) + input="${1#-d=}" + input="${input#--diff=}" + ;; + -d | --diff) + shift + input="${1}" + ;; + -D | --wait-for-display) + display=true + ;; + -s=* | --serial=*) + ANDROID_SERIAL="${1#-s=}" + export ANDROID_SERIAL="${ANDROID_SERIAL#--serial=}" + ;; + -s | --serial) + shift + export ANDROID_SERIAL="${1}" + ;; + *) + echo "${RED}[ ERROR ]${NORMAL} unknown parm ${1}" >&2 + echo "${USAGE}" >&2 + exit 1 + ;; + esac + shift +done + +if [ -n "${input}" -a -z "${output#-}" ]; then + echo "${RED}[ ERROR ]${NORMAL} must specify an -o along with the -d ${input}" >&2 + echo "${USAGE}" >&2 + exit 1 +fi + +USB_SERIAL= +[ -z "${ANDROID_SERIAL}" ] || USB_SERIAL=`find /sys/devices -name serial | + grep usb | + xargs grep -l ${ANDROID_SERIAL}` +USB_ADDRESS= +if [ -n "${USB_SERIAL}" ]; then + USB_ADDRESS=${USB_SERIAL%/serial} + USB_ADDRESS=usb${USB_ADDRESS##*/} +fi + +[ "USAGE: input= BLUE= output= NORMAL= perform_diff < ${output} + +Reports the differences between stdin and \${input}" ] +perform_diff() { + diff -U1 "${input}" - | + # Squash noise, replacing long stretches of changes with trailing ... + sed '/^@@ -1,[0-9]\{4,\} [+]0,0 @@$/ q + /^@@ -0,0 [+]1,[0-9]\{4,\} @@$/ q + /^-\/sys\/module\/[^\/]*\/sections\/[.]bss$/ { + N + /^\(-\/sys\/module\/[^\/]*\/sections\/[.]\)bss\n\1data$/ { + : loop + N + /^-\(\/sys\/module\/[^\/]*\/sections\/[.]\)bss\n-\1data.*\n[+]\1bss[.][^\n]*$/ b loop + s/^-\(\/sys\/module\/[^\/]*\/sections\/[.]\)bss\n-\1data\(.*\n[+]\1bss[.][^\n]*\)\(\n\)\([^\n]*\)$/-\1bss\2\3-\1data\3\4/ + } + }' | + sed '/^[-+ ]\/sys\/module\/[^\/]*$/ { + : loop1 + N + /^ \([^\n]*\)\n[+]\1\/\(coresize\|parameters\)$/ b loop2 + /^\([-+][^\n]*\)\n\1\/\(coresize\|parameters\)$/ { + : loop2 + N + /^\([-+][^\n]*\)\n.*\n\1\/uevent$/ b skip1 + /^ \([^\n]*\)\n.*\n[+]\1\/uevent$/ b skip2 + /^ \([^\n]*\)\n.*\n[+]\1\/[^\n]*$/ b loop2 + /^\([-+][^\n]*\)\n.*\n\1\/[^\n]*$/ b loop2 + h + s/^\([-+][^\n]*\)\n.*\n\1\/[^\n]*\n[^\n]*$/\1\/.../ + s/^ \([^\n]*\)\(\n\).*\n[+]\1\/[^\n]*\n[^\n]*$/ \1\2+\1\/.../ + p + g + s/^.*\n\([^\n]*\)$/\1/ + /^[-+ ]\/sys\/module\/[^\/]*$/ b loop1 + b exit + : skip1 + s/^\([^\n]*\)\n.*\n\1\/uevent$/\1\/.../ + b exit + : skip2 + s/^ \([^\n]*\)\(\n\).*\n[+]\1\/uevent$/\1\2\1\/.../ + : exit + } + } + /^[-+]\/sys\/module\/[^\/]*\/refcnt$/ { + : loop3 + N + /^\([-+][^\n]*\/\)refcnt.*\n\1[^\n]*$/ b loop3 + h + s/^\([-+][^\n]*\/\)\(refcnt\n\).*\n\1[^\n]*\n[^\n]*$/\1\2\1.../ + p + g + s/^.*\n\([^\n]*\)$/\1/ + /^[-+ ]\/sys\/module\/[^\/]*$/ b loop1 + } + /^-\/sys\/module\/[^\/]*\/sections\/[.]\(bss\|data\|rodata\|text\)$/ { + : loop4 + N + /^-\(\/sys\/module\/[^\/]*\/sections\/[.]\(bss\|data\|rodata\|text\)\).*\n[+]\1[.][^\n]*$/ b loop4 + h + s/^-\(\/sys\/module\/[^\/]*\/sections\/[.]\(bss\|data\|rodata\|text\)\)\n.*\(\n\)[+]\1[.][^\n]*\n[^\n]*$/-\1\3+\1.../ + s/^-\(\/sys\/module\/[^\/]*\/sections\/[.]\(bss\|data\|rodata\|text\)\)\n[^\n]*[^.][^.]$/-\1/ + s/^-\(\/sys\/module\/[^\/]*\/sections\/[.]\(bss\|data\|rodata\|text\)\)\(\n[+]\1[.][^\n]*\)\n[^\n]*[^.][^.]$/-\1\3/ + p + g + s/^.*\n\([^\n]*\)$/\1/ + /^-\/sys\/module\/[^\/]*\/sections\/[.]\(bss\|data\|rodata\|text\)$/ b loop4 + /^[-+ ]\/sys\/module\/[^\/]*$/ b loop1 + } + /^ \/sys\/module\/[^\/]*\/sections\/[.]\(bss\|data\|rodata\|text\)[.]/ { + : loop5 + N + /^ \(\/sys\/module\/[^\/]*\/sections\/[.]\(bss\|data\|rodata\|text\)\)[.][^\n].*\n[+]\1[.][^\n]*$/ b loop5 + h + s/^ \(\/sys\/module\/[^\/]*\/sections\/[.]\(bss\|data\|rodata\|text\)\)\([.][^\n]*\n\).*\n[+]\1[.][^\n]*\n[^\n]*$/ \1\3+\1.../ + s/^\( \/sys\/module\/[^\/]*\/sections\/[.]\(bss\|data\|rodata\|text\)[.][^\n]*\)\n[^\n]*[^.][^.]$/\1/ + p + g + s/^.*\n\([^\n]*\)$/\1/ + /^ \/sys\/module\/[^\/]*\/sections\/[.]\(bss\|data\|rodata\|text\)[.]/ b loop5 + /^-\/sys\/module\/[^\/]*\/sections\/[.]\(bss\|data\|rodata\|text\)$/ b loop4 + /^[-+ ]\/sys\/module\/[^\/]*$/ b loop1 + }' | + grep '[-+ ]' || + echo "${BLUE}[ INFO ]${NORMAL} no differences in /dev/ and /sys/ since last device snapshot" >&2 + if [ -s "${output}.probed" -a -s "${input}.probed" ]; then + diff -U1 "${input}.probed" "${output}.probed" | + grep '[-+ ]' || + echo "${BLUE}[ INFO ]${NORMAL} no differences in probed devices and drivers" >&2 + fi + if [ -s "${output}.dmesg" -a -s "${input}.dmesg" ]; then + TMP=`mktemp -d` + trap "rm -rf ${TMP} ; exit 1" 1 2 3 15 + mkfifo ${TMP}/input + sed -n 's@^.*init: Loaded kernel module /lib/modules/@@p' ${input}.dmesg > ${TMP}/input & + sed -n 's@^.*init: Loaded kernel module /lib/modules/@@p' ${output}.dmesg | + diff -U1 ${TMP}/input - | + sed '/^@@ -1,[0-9]\{3,\} [+]0,0 @@$/ q + /^@@ -0,0 [+]1,[0-9]\{3,\} @@$/ q' | + grep '[-+ ]' || + echo "${BLUE}[ INFO ]${NORMAL} no differences in module load order" >&2 + wait + rm -rf ${TMP} + trap "exit 1" 1 2 3 15 + fi + if [ -s "${output}.config" -a -s "${input}.config" ]; then + diff -U1 "${input}.config" "${output}.config" | + grep '[-+ ]' || + echo "${BLUE}[ INFO ]${NORMAL} no differences in config" >&2 + fi + if [ -s "${output}.getprop" -a -s "${input}.getprop" ]; then + TMP=`mktemp -d` + trap "rm -rf ${TMP} ; exit 1" 1 2 3 15 + sed -n 's/[[]ro[.]boottime[.]\([^]]*\)[]]: [[]\([0-9]*\)[]]$/\1 \2/p' "${input}.getprop" >${TMP}/old.getprop + sed -n 's/[[]ro[.]boottime[.]\([^]]*\)[]]: [[]\([0-9]*\)[]]$/\2 \1/p' "${output}.getprop" | + sort -un | + while read monotonic action; do + [ -n "${action}" ] || continue + [ -n "${monotonic}" ] || continue + old=`sed -n "s/^${action} //p" ${TMP}/old.getprop` + [ -n "${old}" ] || continue + old="$((${monotonic} - ${old}))" + [ -n "${old}" -a 0 != "${old}" ] || continue + sign=+ + if [ -z "${old##-*}" ]; then + sign=- + old=${old#-} + fi + printf "${action}\t${sign}%u.%09us\n" $((${old} / 1000000000)) $((${old} % 1000000000)) + done | + grep '[0-9][.][0-9]' || + echo "${BLUE}[ INFO ]${NORMAL} no differences in boottime" >&2 + rm -rf ${TMP} + trap "exit 1" 1 2 3 15 + fi +} + +if [ -n "${output#-}" -a -s "${output}" ]; then + if ! ${force}; then + if [ -n "${input}" ]; then + perform_diff <${output} + exit 0 + fi + echo "${RED}[ ERROR ]${NORMAL} will not overwrite existing content" >&2 + exit 1 + fi +fi + +if [ -z "${filter}" ]; then + if inFastboot; then + echo "${RED}[ ERROR ]${NORMAL} in fastboot mode" >&2 + exit 1 + fi + if inRecovery; then + echo "${RED}[ ERROR ]${NORMAL} in recovery mode" >&2 + echo "${BLUE}[ INFO ]${NORMAL} waiting 2 more minutes for adb ..." + adb_wait 2m || exit 1 + fi + if ! inAdb; then + any_wait 2m + if ! inAdb; then + if [ -n "${ANDROID_SERIAL}" ]; then + echo "${RED}[ ERROR ]${NORMAL} not in adb mode" >&2 + exit 1 + fi + COUNT=`adb devices &2 + exit 1 + fi + fi + fi + counter=0 + while ${display}; do + if inFastboot; then + fastboot reboot + elif inAdb; then + if [ "1" = "`get_property sys.boot_completed`" ]; then + break + fi + fi + counter=$((${counter} + 1)) + if [ ${counter} -gt 120 ]; then + break + fi + sleep 1 + done +fi + +if [ -n "${filter}" ]; then + cat ${filter} +else + adb_su find /dev /sys /dev/null | + if [ -n "${output}" -a X"${output}" != X"-" ]; then + # Generate ${output}.probed: + # + # Report all the instantiated devices and which driver is responsible. + # If drivers do not have any instantiated devices, report them at end. + # Drivers are of a form driver::, if is + # empty, it is short hand for platform. + # + DRIVERS= + FOUND= + TMP=`mktemp -d` + trap "rm -rf ${TMP} ; exit 1" 1 2 3 15 + mkfifo ${TMP}/99 + ( + tee ${TMP}/99 | + sed -n \ + -e 's@/sys/devices/platform/.*/driver$@&@p' \ + -e 's@/sys/bus/\([^/]*\)/drivers/\([^/]*\)$@driver:\1:\2@p' | + sort -u + echo DONE + ) | ( + cat ${TMP}/99 & + while read driver; do + [ -n "${driver}" ] || continue + case ${driver} in + DONE) + [ -n "${DRIVERS}" ] || break + if [ -z "${FOUND}" ]; then + echo "${DRIVERS}" + else + echo "${DRIVERS}" | + grep -ve "${FOUND}\)\$" || + true + fi | + sed 's/driver:platform:/driver::/' + break + ;; + driver:*) + [ -z "${DRIVERS}" ] || DRIVERS="${DRIVERS} +" + DRIVERS="${DRIVERS}${driver}" + ;; + *) + INSTANCE="${driver#/sys/devices/platform/}" + INSTANCE="${INSTANCE%/driver}" + DRIVER="`adb_su ls -l ${driver} /dev/null | + sed -n 's@.* -> .*/bus/\([^/]*\)/drivers/\(.*\)@driver:\1:\2@p'`" + if [ -z "${DRIVER}" ]; then + echo "${INSTANCE}" + else + if [ -z "${FOUND}" ]; then + FOUND="^\(${DRIVER}" + else + FOUND="${FOUND}\|${DRIVER}" + fi + DRIVER="${DRIVER#driver:}" + echo "${INSTANCE} -> driver:${DRIVER#platform}" + fi + ;; + esac + done >"${output}.probed" + wait + ) + rm -rf ${TMP} + trap 'exit 1' 1 2 3 15 + [ -s "${output}.probed" ] || rm -f "${output}.probed" + # Generate ${output}.dmesg + adb_su dmesg /dev/null >"${output}.dmesg" + [ -s "${output}.dmesg" ] || rm -f "${output}.dmesg" + # Generate ${output}.config if kernel provides it. + adb_su cat /proc/config.gz /dev/null | + gunzip >"${output}.config" + [ -s "${output}.config" ] || rm -f "${output}.config" + # Generate ${output}.getprop + adb $QD shell su root getprop /dev/null | + sort -u >"${output}.getprop" + [ -s "${output}.getprop" ] || rm -f "${output}.getprop" + fi +fi | + grep -v '^\(/dev/ashmem\|/dev/block/mapper/by-uuid/\|/dev/fscklogs/log\)' | + grep -v '^\(/dev/socket/location/\(socket_hal\|cld80211\)\|/sys/class/hwmon/\)' | + grep -v '^/dev/socket/netmgr/netmgr_connect_socket$' | + grep -v '^/dev/socket/qdma/qdma-\(file\|campmgr\)-s$' | + grep -v '^/dev/socket/qmux_radio/' | + grep -v '^/dev/socket/vendor_wpa_wlan' | + grep -v '^/dev/socket/wifihal/wifihal_ctrlsock' | + grep -v '^/sys/dev/char/[0-9]*:[0-9]*$' | + grep -v '^/sys/devices/virtual/thermal/tz-by-name/wsatz[.][0-9]*$' | + grep -v '^/sys/kernel/debug/tracing/instances/wifi' | + grep -v '^/sys/kernel/debug/\(wlan\|WMI\|p2p\|msm-bus-dbg/client-data/npu_bwmon_cdsp\)' | + grep -v '^/sys/class/net/\(wlan\|p2p\)' | + grep -v '^/sys/class/ieee80211/phy' | + grep -v '^/sys/devices/platform/soc/[0-9a-f]*.qcom,icnss/\(net\|firmware\|ieee80211\)' | + grep -v '^/sys/devices/platform/soc/[0-9a-f]*.*@[0-9][0-9]*-cam_\(vana\|vdig\|vio\|clk\)$' | + grep -v '^/sys/kernel/wlan/' | + grep -v '^/sys/kernel/slab/\(f2fs_fsync_inode_entry\|zs\|:[0-9a-f]\{0,7\}\)' | + grep -v '^/sys/devices/system/cpu/cpufreq/policy[0-9]*/schedutil' | + grep -v '/__trace_printk_fmt$' | + sed "s@\(/regulator[.]\)[0-9][0-9]*\$@\1${NUM}@ + s@\(/regulator[.]\)[0-9][0-9]*\(/\|-SUPPLY\)@\1${NUM}\2@g + s@\(/vppservice_\)[0-9][0-9]*\$@\1${NUM}@ + s@\(/msi_irqs/\)[0-9][0-9]*\$@\1${NUM}@ + s@\(/sys/class/rfkill/rfkill\)[0-9][0-9]*@\1${NUM}@ + s@\(/sys/class\|/sys/devices/platform/soc/[0-9a-f]*[.]qcom,gpi-dma\)\(/dma/dma\)[0-9]*\(chan[0-9]*\(\|/.*\)\)\$@\1\2${NUM}\3@ + s@\(/i2c-\)\([0-9]\)\(/[^/]*/i2c-\)\2\(\|/.*\)\$@\1${NUM}\3${NUM}\4@ + s@\(/i2c-\)[0-9]\(\|/[^/]*\)\$@\1${NUM}\2@ + s@-\([0-9]\)/\1\(-[0-9][0-9][0-9][0-9]/\)@-${NUM}/${NUM}\2@g + s@/[0-9]\(-[0-9][0-9][0-9][0-9]\(\|-vdd\|-vdd/.*\|-avdd\|-avdd/.*\|-vio\|-vio/.*\|/.*\)\)\$@${NUM}\1@ + s@\(/sys/devices/platform/soc/[0-9a-z]*[.]qcom,[^/]*/subsys\)[0-9][0-9]*@\1${NUM}@ + s@\(rpmsg_chrdev[.]0[.]0/rpmsg/rpmsg_ctrl\)[0-9][0-9]*@\1${NUM}@ + s@\(/sys/devices/virtual/kgsl/kgsl/pagetables/\)[0-9][0-9]*@\1${NUM}@ + s@\(/sys/devices/virtual/kgsl/kgsl/proc/\)[0-9][0-9]*@\1${NUM}@ + s@\(/sys/devices/virtual/thermal/thermal_zone\)[0-9][0-9]*@\1${NUM}@ + s@\(/sys/devices/virtual/net/\)\(r_\|\)\(rmnet_data\)[0-9][0-9]*\(/queues/tx-\)[0-9][0-9]*@\1\3${NUM}\4${NUM}@ + s@\(/sys/devices/platform/soc/soc:qcom,msm-audio-apr/soc:qcom,msm-audio-apr:qcom,q6core-audio/soc:qcom,msm-audio-apr:qcom,q6core-audio:lpi_pinctrl[@][0-9a-f]*/gpiochip\)[0-9][0-9]*@\1${NUM}@ + s@\(/sys/kernel/debug/adsprpc/\(adsp\|cdsp\|ssc\)rpcd_\)[0-9][0-9]*\$@\1${NUM}@ + s@\(/sys/kernel/debug/binder/proc/\)[0-9][0-9]*\$@\1${NUM}@ + s@\(/sys/kernel/debug/kgsl/proc/\)[0-9][0-9]*@\1${NUM}@ + s@\(/sys/kernel/debug/kgsl/kgsl-[0-9a-f][0-9a-f]*/ctx/\)[0-9][0-9]*@\1${NUM}@ + s@\(/sys/module/[^/]*/sections/.*[.]llvm[.]\)[0-9][0-9]*\$@\1${NUM}@ + s@\(/sys/kernel/debug/msm_cvp/core0/inst_\)[0-9a-f][0-9a-f]*@\1${NUM}@ + s@\(/sys/kernel/iommu_groups/\)[0-9][0-9]*@\1${NUM}@ + s@\(/devices/soc:qcom,msm_fastrpc:qcom,msm_fastrpc_compute_cb\)[0-9][0-9]*@\1${NUM}@ + s@/mmc[0-9][0-9]*\($\|/\|:\)@/mmc${NUM}\1@ + s@\(/sys/kernel/irq/\)[0-9][0-9]*@\1${NUM}@" | + if [ -n "${output}" -a X"${output}" != X"-" ]; then + sort -u | + if [ -n "${input}" ]; then + tee "${output}" | perform_diff + else + cat - >"${output}" + fi + [ -s "${output}" ] || rm -f "${output}" + else + TMP=`mktemp -d` + trap "rm -rf ${TMP} ; exit 1" 1 2 3 15 + if [ X"${output}" = X"-" ]; then + mkfifo ${TMP}/99 + echo "------------------------> cut dump < ------------------------" + sort -u | + tee ${TMP}/99 | + ( + cat ${TMP}/99 & + sed -n 's@/sys/devices/platform/\(.*\)/driver$@\1@p' >${TMP}/probed + wait + rm ${TMP}/99 + ) + else + sort -u + fi + if [ -s "${TMP}/probed" ]; then + echo "-----------------------> cut probed <------------------------" + cat ${TMP}/probed + fi + rm -rf ${TMP} + trap "exit 1" 1 2 3 15 + fi +if [ -z "${filter}" -a X"${output}" = X"-" ]; then + echo "------------------------> cut dmesg <------------------------" + adb_su dmesg /dev/null + echo "-----------------------> cut .config <-----------------------" + adb_su cat /proc/config.gz /dev/null | gunzip +fi diff --git a/gki/download_from_ci b/gki/download_from_ci new file mode 100755 index 0000000000..62f74d3b6e --- /dev/null +++ b/gki/download_from_ci @@ -0,0 +1,575 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2019-2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import argparse +import collections +import functools +import glob +import json +import multiprocessing +import os +import pathlib +import re +import shlex +import shutil +import subprocess +import sys +import tempfile +import urllib.request + +from pathlib import Path + +sys.path.append(str(Path(os.path.dirname(__file__)) / ".." / "abi")) +from abitool import get_abi_tool + +BASE_URL = "https://ci.android.com/builds/submitted/{build_id}/{target}/latest/raw" +ARM_TARGETS = ["u-boot_qemu_arm"] +ARM64_TARGETS = ["kernel_aarch64", "kernel_debug_aarch64", + "u-boot_crosvm_aarch64", "u-boot_qemu_aarch64"] +X86_64_TARGETS = ["kernel_x86_64", "kernel_debug_x86_64", + "u-boot_crosvm_x86_64", "u-boot_qemu_x86_64"] +GIT_COMMIT_LOG_LENGTH = 100 + +# Arguments from argparse. +_args = None + +# Multiprocessing pool. Pool creation must happen after the worker fn is defined. +_pool = None + +def download(url_base, dest_dir, filename, dest_filename = None): + dest_filename = os.path.join(dest_dir, dest_filename or filename) + if not os.path.exists(dest_filename): + print("Downloading %s to %s" % (filename, dest_dir)) + try: + urllib.request.urlretrieve(os.path.join(url_base, filename), dest_filename) + except urllib.error.HTTPError as e: + print("Could not download %s: %s" % (filename, e)) + +def parse_args(): + global _args + + parser = argparse.ArgumentParser() + parser.add_argument( + "build_id", + type=int, + help="the build id to download the build for, e.g. 6148204") + parser.add_argument( + "build_target", + nargs="?", + type=str, + help='the build target to download, e.g. "kernel_aarch64"') + parser.add_argument( + "-u", + "--update-gki", + action="store_true", + help="update GKI kernel prebuilts in Android platform (set $ANDROID_BUILD_TOP)") + parser.add_argument( + "--update-u-boot", + action="store_true", + help="update U-Boot prebuilts in Android platform (set $ANDROID_BUILD_TOP)") + parser.add_argument( + "-r", + "--repo-branch", + action="store_true", + help="create branches in Android repo that can be uploaded together (set $ANDROID_BUILD_TOP)") + parser.add_argument( + "-b", + "--bug", + type=int, + default=0, + help="bug number for kernel prebuilt update commit") + + _args = parser.parse_args() + + if _args.update_gki or _args.update_u_boot: + if _args.build_target is not None: + raise Exception("build_target cannot be specified when updating prebuilts") + if "ANDROID_BUILD_TOP" not in os.environ: + raise Exception("$ANDROID_BUILD_TOP must be set to update prebuilts") + if not _args.bug: + raise Exception("A bug number must be provided when updating prebuilts") + +def list_artifacts(url_base): + # Download BUILD_INFO to get the file list + url = os.path.join(url_base, "BUILD_INFO") + response = urllib.request.urlopen(url) + data = response.read().decode("utf-8") + return json.loads(data)["target"]["dir_list"] + +def download_files(url_base, files): + """Download multiple files in url_base using a multiprocessing pool.""" + for f in files: + dirname = os.path.dirname(f) + if dirname and not os.path.isdir(dirname): + os.makedirs(dirname) + + func = functools.partial(download, url_base, "") + _pool.map(func, files) + +def get_kernel_dir(): + """ Return the path of the kernel source directory in the same repo as this script.""" + # download_from_ci lives in /build/gki + return pathlib.Path(os.path.abspath(__file__)).parents[2].joinpath('common') + +def get_u_boot_dir(): + """ Return the path of the u-boot source directory in the same repo as this script.""" + # download_from_ci lives in /build/gki + return pathlib.Path(os.path.abspath(__file__)).parents[2].joinpath('u-boot') + +def get_build_kernel_version(data): + """Given a build's BUILD_INFO in json format, return the kernel version as a string.""" + branch = json.loads(data)["branch"] + if branch == "aosp_kernel-common-android-mainline": + return "mainline" + else: + pattern = "android(\d\d)?-(?P\d+\.\d+|\w\+)(-stable)?$" + result = re.search(pattern, branch) + if not result: + raise Exception("Could not determine kernel version in branch " + branch) + return result.group('version') + +def get_git_log(old_sha, new_sha, path=get_kernel_dir()): + """Return a string containing the git log between two commits.""" + gitlog = "git log --first-parent -{} --oneline --no-decorate {}..{}".format(GIT_COMMIT_LOG_LENGTH, + old_sha,new_sha) + gitlog += ":\n" + subprocess.check_output(shlex.split(gitlog), text=True, + cwd=path) + "\n" + return gitlog + +def commit_prebuilts(output_dir, gitlog, name="kernel"): + commit_message = ("Update {} to builds {}\n" + "\n" + "{}" + "\n" + "Test: treehugger\n" + "Bug: {}\n").format(name, _args.build_id, gitlog, _args.bug) + subprocess.check_call(["git", "add", "."], cwd=output_dir) + subprocess.check_call(["git", "commit", "-a", "-m", commit_message], cwd=output_dir) + if _args.repo_branch: + branch_name = "kernel-update-b{}-{}".format(_args.bug, _args.build_id) + subprocess.check_call(["repo", "start", "--head", branch_name], cwd=output_dir) + print("Changes committed to local Android branch: {}.".format(branch_name)) + print("Use the following command from your Android repository to upload your changes:") + print(" repo upload -t --br {}".format(branch_name)) + +def get_binary_kernel_version(path): + """Returns a tuple of strings of the x.y.z (LTS) version and the SHA.""" + rawdata = subprocess.check_output(["grep", "-a", "Linux version ", path]) + pattern = b"Linux version (?P(?P[0-9]+[.][0-9]+[.][0-9]+)(-rc\d+)?(-\w+(-\d+)?)?-\d+-g(?P\w+)-ab\d+) \(.*@.*\) \((?P.*)\) .*\n" + result = re.search(pattern, rawdata) + if not result: + raise Exception("Could not determine version in kernel binary " + path) + return (result.group("version").decode('ascii'), + result.group("sha").decode('ascii')) + +def get_binary_u_boot_version(path): + """Returns a tuple of strings of the YYYY.MM(-rcXX) version and the SHA.""" + rawdata = subprocess.check_output(["grep", "-a", "U-Boot\s[0-9]*.[0-9]*-.*", path]) + pattern = b"U-Boot (?P(?P[0-9]+[.][0-9]+)(-rc\d+)?(-\d+)?-g(?P\w+)(-dirty)?) .*" + result = re.search(pattern, rawdata) + if not result: + raise Exception("Could not determine version in U-Boot binary " + path) + return (result.group("version").decode('ascii'), + result.group("sha").decode('ascii')) + +def write_prebuilt_info(output_dir): + with open(os.path.join(output_dir, "prebuilt-info.txt"), "w") as prebuilt_info: + json.dump({"kernel-build-id": _args.build_id}, prebuilt_info, indent=4) + prebuilt_info.write("\n") + +def repackage_kernels(build_is_arm64, output_dir, version, suffix = ""): + uncompressed_kernel = os.path.join(output_dir, "kernel-" + version + suffix) + lz4_kernel = os.path.join(output_dir, "kernel-" + version + "-lz4" + suffix) + if build_is_arm64: + shutil.move(os.path.join(output_dir, "Image"), uncompressed_kernel) + # older platform versions did not need compressed kernels + if not version in ["4.14"]: + subprocess.check_call(["gzip -nc " + uncompressed_kernel + " > " + + os.path.join(output_dir, "kernel-" + + version + "-gz" + suffix)], + shell=True) + subprocess.check_call(["lz4", "-f", "-l", "-12", "--favor-decSpeed", + uncompressed_kernel, lz4_kernel]) + else: + shutil.move(os.path.join(output_dir, "bzImage"), + os.path.join(output_dir, "kernel-" + version + suffix)) + + +def get_or_update_abi_definition(target, output_dir): + url_base = BASE_URL.format(build_id=_args.build_id, target=target) + files = list_artifacts(url_base) + + output_dir = Path(output_dir) + abi_prop = "abi.prop" + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + + props = {} + if abi_prop in files: + # First, download the abi.prop to see if we can reuse something + download(url_base, tmp, abi_prop) + with open(tmp / abi_prop) as prop: + props.update(line.split("=", 1) for line in prop.read().splitlines()) + + # Now see whether there is a symbol list to consider + symbol_list = props.get("KMI_SYMBOL_LIST", None) + if symbol_list is not None: + download(url_base, tmp, symbol_list) + shutil.copyfile(tmp / symbol_list, output_dir / "abi_symbol_list") + + # If there is an existing XML representation, we do not generate one, so + # let's try that + abi_xml = props.get("KMI_DEFINITION", None) + if abi_xml is not None: + download(url_base, tmp, abi_xml) + else: + abi_xml = "abi.xml" + # So there was none, let's download the binaries then + abi_files = [x for x in files if re.search(".+\.ko$|^abi", x) + ] + ["vmlinux"] + func = functools.partial(download, url_base, tmp) + _pool.map(func, abi_files) + + get_abi_tool().dump_kernel_abi(tmp, tmp / abi_xml, + tmp / symbol_list if symbol_list else None) + + shutil.copyfile(tmp / abi_xml, output_dir / "abi.xml") + +def download_kernel(target, version, output_dir): + if not os.path.isdir(output_dir): + os.mkdir(output_dir) + + build_is_arm64 = target in ARM64_TARGETS + files = ["Image"] if build_is_arm64 else ["bzImage"] + files.append("System.map") + + url_base = BASE_URL.format(build_id=_args.build_id, target=target) + func = functools.partial(download, url_base, output_dir) + _pool.map(func, files) + + if target.startswith("kernel_debug_"): + suffix = "-allsyms" + shutil.move(os.path.join(output_dir, "System.map"), + os.path.join(output_dir, "System.map" + suffix)) + else: + suffix = "" + repackage_kernels(build_is_arm64, output_dir, version, suffix) + +def download_kernel_modules(target, output_dir, fetch_initramfs=False): + if not os.path.isdir(output_dir): + os.mkdir(output_dir) + + url_base = BASE_URL.format(build_id=_args.build_id, target=target) + files = list_artifacts(url_base) + files_to_fetch = [x for x in files if x.endswith((".dtb", ".ko"))] + if fetch_initramfs: + files_to_fetch.append("initramfs.img") + + func = functools.partial(download, url_base, output_dir) + _pool.map(func, files_to_fetch) + +def update_android_4_1x_pq(output_dir, version): + output_dir = os.path.join(output_dir, "device", "google", "cuttlefish_kernel") + + arm64_dirname = "%s-arm64" % version + arm64_dir = os.path.join(output_dir, arm64_dirname) + x86_64_dirname = "%s-x86_64" % version + x86_64_dir = os.path.join(output_dir, x86_64_dirname) + + arm64_kernel = os.path.join(arm64_dir, "kernel-%s" % version) + (old_version, old_sha) = get_binary_kernel_version(arm64_kernel) + + subprocess.check_call(["git", "rm", "-rf", arm64_dirname, x86_64_dirname], + cwd=output_dir) + + download_kernel("kernel_aarch64", version, arm64_dir) + download_kernel("kernel_x86_64", version, x86_64_dir) + + (new_version, new_sha) = get_binary_kernel_version(arm64_kernel) + + gitlog = get_git_log(old_sha, new_sha) + commit_prebuilts(output_dir, gitlog) + +def update_android_4_14_p(output_dir): + return update_android_4_1x_pq(output_dir, "4.14") + +def update_android_4_19_q(output_dir): + return update_android_4_1x_pq(output_dir, "4.19") + +def update_android_4_19_r(output_dir): + old_sha = None + + arm64_dir = os.path.join(output_dir, "kernel", "prebuilts", "4.19", "arm64") + arm64_kernel = os.path.join(arm64_dir, "kernel-4.19") + if os.path.isfile(arm64_kernel): + (old_version, old_sha) = get_binary_kernel_version(arm64_kernel) + + # arm64 GKI kernels and kernel modules + subprocess.check_call(["git", "rm", "-rf", "*"], cwd=arm64_dir) + download_kernel("kernel_debug_aarch64", "4.19" , arm64_dir) + download_kernel("kernel_aarch64", "4.19", arm64_dir) + download_kernel_modules("kernel_aarch64", arm64_dir) + get_or_update_abi_definition("kernel_aarch64", arm64_dir) + write_prebuilt_info(arm64_dir) + (new_version, new_sha) = get_binary_kernel_version(arm64_kernel) + if old_sha is not None: + gitlog = get_git_log(old_sha, new_sha) + else: + gitlog = "Initial prebuilts at commit {}\n".format(new_sha) + commit_prebuilts(arm64_dir, gitlog) + +def update_android11_5_4(output_dir): + output_dir = os.path.join(output_dir, "device", "google", "cuttlefish_kernel") + + arm64_dirname = "5.4-arm64" + arm64_dir = os.path.join(output_dir, arm64_dirname) + x86_64_dirname = "5.4-x86_64" + x86_64_dir = os.path.join(output_dir, x86_64_dirname) + + arm64_kernel = os.path.join(arm64_dir, "kernel-5.4") + (old_version, old_sha) = get_binary_kernel_version(arm64_kernel) + + subprocess.check_call(["git", "rm", "-rf", arm64_dirname, x86_64_dirname], + cwd=output_dir) + + # arm64 GKI kernel and kernel modules + download_kernel("kernel_debug_aarch64", "5.4", arm64_dir) + download_kernel("kernel_aarch64", "5.4", arm64_dir) + download_kernel_modules("kernel_aarch64", arm64_dir) + download_kernel_modules("kernel_cf_aarch64", arm64_dir) + get_or_update_abi_definition("kernel_aarch64", arm64_dir) + + # x86_64 GKI kernel and kernel modules + download_kernel("kernel_debug_x86_64", "5.4", x86_64_dir) + download_kernel("kernel_x86_64", "5.4", x86_64_dir) + download_kernel_modules("kernel_x86_64", x86_64_dir) + download_kernel_modules("kernel_cf_x86_64", x86_64_dir) + get_or_update_abi_definition("kernel_x86_64", x86_64_dir) + + (new_version, new_sha) = get_binary_kernel_version(arm64_kernel) + + gitlog = get_git_log(old_sha, new_sha) + commit_prebuilts(output_dir, gitlog) + +def update_androidx_virtual_device_modules(gitlog, output_dir, version): + mods_dir = os.path.join(output_dir, "kernel", "prebuilts", "common-modules", + "virtual-device") + ver_mods_dir = os.path.join(mods_dir, version) + arm64_mods_dir = os.path.join(ver_mods_dir, "arm64") + x86_64_mods_dir = os.path.join(ver_mods_dir, "x86-64") + + try: + subprocess.check_call(["git", "rm", "-rf", "*"], cwd=arm64_mods_dir) + subprocess.check_call(["git", "checkout", "HEAD", "Android.bp"], cwd=arm64_mods_dir) + except subprocess.CalledProcessError: + pass + + try: + subprocess.check_call(["git", "rm", "-rf", "*"], cwd=x86_64_mods_dir) + subprocess.check_call(["git", "checkout", "HEAD", "Android.bp"], cwd=x86_64_mods_dir) + except subprocess.CalledProcessError: + pass + + download_kernel_modules("kernel_virt_aarch64", arm64_mods_dir, fetch_initramfs=True) + download_kernel_modules("kernel_virt_x86_64", x86_64_mods_dir, fetch_initramfs=True) + + commit_prebuilts(arm64_mods_dir, gitlog) + commit_prebuilts(x86_64_mods_dir, gitlog) + +def update_androidx_cf(gitlog, output_dir, version): + i686_cf_dirname = version + "-i686" + + cf_dir = os.path.join(output_dir, "device", "google", "cuttlefish_prebuilts", + "kernel") + i686_cf_dir = os.path.join(cf_dir, i686_cf_dirname) + + subprocess.check_call(["git", "rm", "-rf", i686_cf_dirname], cwd=cf_dir) + + download_kernel("kernel_virt_i686", version, i686_cf_dir) + download_kernel_modules("kernel_virt_i686", i686_cf_dir) + + commit_prebuilts(cf_dir, gitlog) + +def update_androidx(output_dir, version): + arm64_dir = os.path.join(output_dir, "kernel", "prebuilts", version, "arm64") + x86_64_dir = os.path.join(output_dir, "kernel", "prebuilts", version, + "x86_64") + arm64_kernel = os.path.join(arm64_dir, "kernel-" + version) + (old_version, old_sha) = get_binary_kernel_version(arm64_kernel) + + # arm64 GKI kernels and kernel modules + subprocess.check_call(["git", "rm", "-rf", "*"], cwd=arm64_dir) + subprocess.check_call(["git", "checkout", "HEAD", "Android.bp"], cwd=arm64_dir) + download_kernel("kernel_debug_aarch64", version, arm64_dir) + download_kernel("kernel_aarch64", version, arm64_dir) + download_kernel_modules("kernel_aarch64", arm64_dir) + get_or_update_abi_definition("kernel_aarch64", arm64_dir) + write_prebuilt_info(arm64_dir) + (new_version, new_sha) = get_binary_kernel_version(arm64_kernel) + gitlog = get_git_log(old_sha, new_sha) + commit_prebuilts(arm64_dir, gitlog) + + # x86_64 GKI kernels and kernel modules + subprocess.check_call(["git", "rm", "-rf", "*"], cwd=x86_64_dir) + subprocess.check_call(["git", "checkout", "HEAD", "Android.bp"], cwd=x86_64_dir) + download_kernel("kernel_debug_x86_64", version, x86_64_dir) + download_kernel("kernel_x86_64", version, x86_64_dir) + download_kernel_modules("kernel_x86_64", x86_64_dir) + get_or_update_abi_definition("kernel_x86_64", x86_64_dir) + write_prebuilt_info(x86_64_dir) + commit_prebuilts(x86_64_dir, gitlog) + + update_androidx_virtual_device_modules(gitlog, output_dir, version) + update_androidx_cf(gitlog, output_dir, version) + +def update_android_mainline(output_dir): + arm64_dir = os.path.join(output_dir, "kernel", "prebuilts", "mainline", + "arm64") + arm64_kernel = os.path.join(arm64_dir, "kernel-mainline-allsyms") + (old_version, old_sha) = get_binary_kernel_version(arm64_kernel) + + subprocess.check_call(["git", "rm", "-rf", "*"], cwd=arm64_dir) + download_kernel("kernel_debug_aarch64", "mainline", arm64_dir) + download_kernel_modules("kernel_aarch64", arm64_dir) + write_prebuilt_info(arm64_dir) + get_or_update_abi_definition("kernel_aarch64", arm64_dir) + + (new_version, new_sha) = get_binary_kernel_version(arm64_kernel) + + gitlog = get_git_log(old_sha, new_sha) + commit_prebuilts(arm64_dir, gitlog) + +def update_gki(): + url_base = BASE_URL.format(build_id=_args.build_id, target="kernel_aarch64") + url = os.path.join(url_base, "BUILD_INFO") + response = urllib.request.urlopen(url) + data = response.read().decode("utf-8") + branch = json.loads(data)["branch"] + + output_dir = os.environ["ANDROID_BUILD_TOP"] + + if branch == "aosp_kernel-p-common-android-4.14": + update_android_4_14_p(output_dir) + elif branch == "aosp_kernel-q-common-android-4.19": + update_android_4_19_q(output_dir) + elif branch == "aosp_kernel-common-android-4.19-stable": + update_android_4_19_r(output_dir) + elif branch == "aosp_kernel-common-android11-5.4": + update_android11_5_4(output_dir) + elif branch == "aosp_kernel-common-android-mainline": + update_android_mainline(output_dir) + else: + update_androidx(output_dir, get_build_kernel_version(data)) + +def download_u_boot(target, output_dir): + if not os.path.isdir(output_dir): + os.mkdir(output_dir) + + build_is_arm = target in ARM_TARGETS + ARM64_TARGETS + files = ["u-boot.bin"] if build_is_arm else ["u-boot.rom"] + files.append("System.map") + + url_base = BASE_URL.format(build_id=_args.build_id, target=target) + func = functools.partial(download, url_base, output_dir) + _pool.map(func, files) + +def download_u_boot_tools(target, output_dir): + if not os.path.isdir(output_dir): + os.mkdir(output_dir) + + files = ["mkenvimage", "mkimage"] + + url_base = BASE_URL.format(build_id=_args.build_id, target=target) + func = functools.partial(download, url_base, output_dir) + _pool.map(func, files) + + subprocess.check_call(["chmod", "a+x"] + files, cwd=output_dir) + +def update_u_boot_mainline(output_dir): + cf_dir = os.path.join(output_dir, "device", "google", "cuttlefish_prebuilts") + bootloader_dir = os.path.join(cf_dir, "bootloader") + + crosvm_aarch64_dir = os.path.join(bootloader_dir, "crosvm_aarch64") + crosvm_x86_64_dir = os.path.join(bootloader_dir, "crosvm_x86_64") + qemu_aarch64_dir = os.path.join(bootloader_dir, "qemu_aarch64") + qemu_arm_dir = os.path.join(bootloader_dir, "qemu_arm") + qemu_x86_64_dir = os.path.join(bootloader_dir, "qemu_x86_64") + + crosvm_aarch64_u_boot = os.path.join(crosvm_aarch64_dir, "u-boot.bin") + + (old_version, old_sha) = get_binary_u_boot_version(crosvm_aarch64_u_boot) + + subprocess.check_call(["git", "rm", "-rf", "*"], cwd=crosvm_aarch64_dir) + download_u_boot("u-boot_crosvm_aarch64", crosvm_aarch64_dir) + subprocess.check_call(["git", "rm", "-rf", "*"], cwd=crosvm_x86_64_dir) + download_u_boot("u-boot_crosvm_x86_64", crosvm_x86_64_dir) + subprocess.check_call(["git", "rm", "-rf", "*"], cwd=qemu_aarch64_dir) + download_u_boot("u-boot_qemu_aarch64", qemu_aarch64_dir) + subprocess.check_call(["git", "rm", "-rf", "*"], cwd=qemu_arm_dir) + download_u_boot("u-boot_qemu_arm", qemu_arm_dir) + subprocess.check_call(["git", "rm", "-rf", "*"], cwd=qemu_x86_64_dir) + download_u_boot("u-boot_qemu_x86_64", qemu_x86_64_dir) + + tools_dir = os.path.join(cf_dir, "uboot_tools") + subprocess.check_call(["git", "rm", "-rf", "*"], cwd=tools_dir) + download_u_boot_tools("u-boot_tools", tools_dir) + subprocess.check_call(["git", "checkout", "HEAD", "mkenvimage_mac.sh"], cwd=tools_dir) + + (new_version, new_sha) = get_binary_u_boot_version(crosvm_aarch64_u_boot) + gitlog = get_git_log(old_sha, new_sha, get_u_boot_dir()) + commit_prebuilts(cf_dir, gitlog, "bootloader, u-boot tools") + +def update_u_boot(): + url_base = BASE_URL.format(build_id=_args.build_id, target="u-boot_crosvm_aarch64") + url = os.path.join(url_base, "BUILD_INFO") + response = urllib.request.urlopen(url) + data = response.read().decode("utf-8") + branch = json.loads(data)["branch"] + + output_dir = os.environ["ANDROID_BUILD_TOP"] + + # Right now, we only have the mainline branch + if branch == "aosp_u-boot-mainline": + update_u_boot_mainline(output_dir) + else: + print("Unknown u-boot branch %s for build" % branch) + +def main(): + global _pool + _pool = multiprocessing.Pool(10) + + parse_args() + + if _args.update_gki: + update_gki() + return + + if _args.update_u_boot: + update_u_boot() + return + + url_base = BASE_URL.format(build_id=_args.build_id, target=_args.build_target) + + url = os.path.join(url_base, "BUILD_INFO") + response = urllib.request.urlopen(url) + data = response.read().decode("utf-8") + files = json.loads(data)["target"]["dir_list"] + + download_files(url_base, files) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/gki/find_circular b/gki/find_circular new file mode 100755 index 0000000000..bb5dcc9527 --- /dev/null +++ b/gki/find_circular @@ -0,0 +1,170 @@ +#! /bin/bash +# SPDX-License-Identifier: Apache-2.0 +# +# (c) 2019, Google +progname="${0##*/}" + +USAGE="USAGE: ${progname} [dir] + +Call this when depmod breaks down, or when one needs a list of the symbols +implicated in the circular dependency. + +Search current or dir directory for all kernel modules. Itemize what they +export, and what they import. Discover links and report who fulfills them. +Report any first order circular relationships and the symbols that got us +into the situation. + +Standard output is of the form: + +module1.ko(symbols) -> module2.ko(symbols) -> module1.ko + +Leaves an annotated modules.dep file in the specified directory. + +TBA: higher order circular dependencies" + +if [ X"--help" = X"${1}" -o X"{-h}" = X"${1}" ]; then + echo "${USAGE}" >&2 + exit +fi +DIR=. +if [ 1 = ${#} ]; then + DIR=${1} + shift +fi +if [ 0 != ${#} ]; then + echo "Unexpected Argument: ${*}" >&2 + echo >&2 + echo "${USAGE}" >&2 + exit 1 +fi + +# A _real_ embedded tab character +TAB="`echo | tr '\n' '\t'`" + +# +# Start Work +# + +# Construct our version (symbols annoted as comments) of the modules.dep file. + +TMP=`mktemp -d` + +# Acquire a list of files that are modules +find ${DIR%/} -name '*.ko' >${TMP}/modules + +# Fill symbols with file type symbol +# type is > for requires symbol +# type is < for supplies symbol +# type is pre for requires module (in symbol field) +# type is post for supplies module (in file field) + +# Mine for symbols +xargs nm -o <${TMP}/modules | + sed "s@^${DIR%/}/@@" | + sed -n \ + -e 's/^\([^:]*\):.* U \(.*\)/\1 > \2/p' \ + -e 's/^\([^:]*\):.* [TD] \(.*\)/\1 < \2/p' >${TMP}/symbols +if [ ! -s ${TMP}/symbols ]; then + rm -rf ${TMP} + echo ERROR: failed to find any .ko module files in the specified directory ${DIR%/} + exit 1 +fi + +# Mine for softdep pre + while read file; do + strings ${file} | + sed -n 's/^softdep=pre: \(.*\)/\1/p' | + tr ' ' '\n' | + sed "s@.*@${file#${DIR%/}/} pre &@" + done < ${TMP}/modules | + sort -u >>${TMP}/symbols + +# Mine for softdep post + while read file; do + strings ${file} | + sed -n 's/^softdep=post: \(.*\)/\1/p' | + tr ' ' '\n' | + sed "s@.*@$& post {file#${DIR%/}/}@" + done < ${TMP}/modules | + sort -u >>${TMP}/symbols + +# Translate symbols to annotated modules.dep +( + sed -n 's/ > / /p' ${TMP}/symbols | + while read file symbol ; do + sed -n "s@^\(.*\) < ${symbol}\$@${file}: \1 # ${symbol}@p" ${TMP}/symbols + done + sed -n 's/ pre / /p' ${TMP}/symbols | + while read file softdep ; do + grep "/${softdep}[.]ko\$" ${TMP}/modules | + sed "s@^${DIR%/}/@@" | + while read source; do + echo "${file}: ${source} # pre:${softdep}" + done + done + sed -n 's/ post / /p' ${TMP}/symbols | + while read softdep source ; do + grep "/${softdep}[.]ko\$" ${TMP}/modules | + sed "s@^${DIR%/}/@@" | + while read file; do + echo "${file}: ${source} # post:${softdep}" + done + done +) | + sort -u >${DIR%/}/modules.dep + +# Cleanup (already!) +rm -rf ${TMP} + +# Evaluate first order dependencies from our annotated ${DIR}/modules.dep file. + +# squash adjacent lines with match for first symbols, merge second +merge_second_symbols() { + sed ': loop + N + s/^\(.*(.*) -> .*(\)\(.*\)\().*\)\n\1\(.*\)\3/\1\2,\4\3/ + t loop + h + s/\n.*//p + g + s/^[^\n]*\n// + t loop' +} + +# squash adjacent lines with match for second symbols, merge first +merge_first_symbols() { + sed ': loop + N + s/^\(.*(\)\(.*\)\() -> .*(.*).*\)\n\1\(.*\)\3/\1\2,\4\3/ + t loop + h + s/\n.*//p + g + s/^[^\n]*\n// + t loop' +} + +# squash adjacent lines with identical, but shifted one, circular dependencies +merge_duplicates() { + sed ': loop + N + s/^\(.*\)\((.*)\) -> \(.*\)\((.*)\) -> \1\n\3\4 -> \1\2 -> \3$/\1\2 -> \3\4 -> \1/ + t end + s/\n.*//p + g + s/^[^\n]*\n// + t loop + : end' +} + +# Report the first order circular dependencies +sed 's/://' ${DIR%/}/modules.dep | + while read file source comment symbol; do + sed -n "s@^${source}: ${file} # \(.*\)@${file}(${symbol}) -> ${source}(\1) -> ${file}@p" ${DIR%/}/modules.dep + done | + merge_second_symbols | + merge_first_symbols | + merge_duplicates | + sort -u + +# TBA: second order dependencies diff --git a/gki/instrument_module_init b/gki/instrument_module_init new file mode 100755 index 0000000000..9e9c4bf394 --- /dev/null +++ b/gki/instrument_module_init @@ -0,0 +1,307 @@ +#! /bin/sh +# SPDX-License-Identifier: Apache-2.0 +# +# (c) 2019, Google +progname="${0##*/}" + +# Add instrumentation to module_init and probe functions. + +USAGE="USAGE: ${progname} [dir|file] + +Add debug instrumentation to module_init and probe functions." + +if [ X"--help" = X"${1}" -o X"{-h}" = X"${1}" ]; then + echo "${USAGE}" >&2 + exit +fi +DIR=. +if [ 1 = ${#} ]; then + DIR=${1} + shift +fi +if [ 0 != ${#} ]; then + echo "Unexpected Argument: ${*}" >&2 + echo >&2 + echo "${USAGE}" >&2 + exit 1 +fi + +# A _real_ embedded tab character +TAB="`echo | tr '\n' '\t'`" +# A _real_ embedded escape character +ESCAPE="`echo | tr '\n' '\033'`" +# Colours +RED="${ESCAPE}[38;5;196m" +ORANGE="${ESCAPE}[38;5;255:165:0m" +BLUE="${ESCAPE}[35m" +NORMAL="${ESCAPE}[0m" + +TMP=`mktemp -d` + +# ${file} ${call} and ${symbol} input +SYMBOLS=" " +add_debug_printk() +{ + message="printk(KERN_WARNING \"${file##*/}:" + if [ -z "${call#select}" ]; then + # We need uniqueness for discovery, so instead of truth of __func__ to + # cover any of our mistakes(!), we select the expectation. + message+="${symbol}: ENTER\\\\n\");" + else + message+="%s:${call}: ENTER\\\\n\", __func__);" + fi + if grep "${TAB}${message%%\\*}" ${file} >/dev/null; then + if [ "${SYMBOLS}" == "${SYMBOLS#* ${symbol}:${call#select}}" ]; then + echo INFO: already ${file} instrumented ${call#select} ${symbol} >&2 + SYMBOLS="${SYMBOLS}${symbol}:${call} " + fi + return + fi + if [ "device_initcall" = "${call}" -o "module_init" = "${call}" -o "module_exit" = "${call}" ] && + grep "${TAB}printk(KERN_WARNING \"${file##*/}:%s:\(builtin\|module\)_[a-z]*_driver: ENTER" ${file} >/dev/null; then + if [ "${SYMBOLS}" == "${SYMBOLS#* ${symbol}:${call#select}}" ]; then + echo INFO: maybe already ${file} instrumented ${call#select} ${symbol} >&2 + fi + return + fi + sed "/^\(static \)\{0,1\}.* ${symbol}(/ { + N + s/^\(static \)\{0,1\}\(.* ${symbol}(\)\(void\)\{0,1\}) {[ ${TAB}]*}\(\n\)/\1\2\3)\4{\4${TAB}${message}\4}\4/ + t exit + N + /\/[*][^*]*\$/ { + : comment_loop + N + /\/[*].*[*]\// b comment_done + b comment_loop + : comment_done + /\/[*].*[*]\/\$/ N + } + /printk(KERN_WARNING \"${file##*/}:/ b exit + s/\(\n${TAB}\)\(kfree(\|DBG(\|return[ ${TAB}]\|if (\|platform_driver_\|class_destroy(\|pr_info(\|[a-z_0-9]*_register\(_drivers\|_simple\)\{0,1\}(\|[a-z_0-9]*_\(un\|de\)register\(_drivers\|_simple\)\{0,1\}(\|[a-z+_0-9]*_driver(\)/\1${message}&/ + t exit + N + /printk(KERN_WARNING \"${file##*/}:/ b exit + s/\(\n${TAB}\)\(DBG(\|return[ ${TAB}]\|if (\|platform_driver_\|pr_info(\|[a-z_0-9]*_register\(_drivers\|_simple\)\{0,1\}(\|[a-z_0-9]*_\(un\|de\)register\(_drivers\|_simple\)\{0,1\}(\)/\1${message}&/ + t exit + s/\(static void .*\)\(\n${TAB}\)\([a-z].*\)\2}\$/\1\2${message}\2\3\2}/ + t exit + N + /printk(KERN_WARNING \"${file##*/}:/ b exit + s/\(.*\)\(\n${TAB}\)\(.*\n\)\2/&${message}\2/ + t exit + N + /printk(KERN_WARNING \"${file##*/}:/ b exit + s/\(.*\)\(\n${TAB}\)\(.*\n\)\2/&${message}\2/ + t exit + N + /printk(KERN_WARNING \"${file##*/}:/ b exit + s/\(.*\)\(\n${TAB}\)\(.*\n\)\2/&${message}\2/ + t exit + N + /printk(KERN_WARNING \"${file##*/}:/ b exit + s/\(.*\)\(\n${TAB}\)\(.*\n\)\2/&${message}\2/ + t exit + N + /printk(KERN_WARNING \"${file##*/}:/ b exit + s/\(.*\)\(\n${TAB}\)\(.*\n\)\2/&${message}\2/ + t exit + N + /printk(KERN_WARNING \"${file##*/}:/ b exit + s/\(.*\)\(\n${TAB}\)\(.*\n\)\2/&${message}\2/ + t exit + N + /printk(KERN_WARNING \"${file##*/}:/ b exit + s/\(.*\)\(\n${TAB}\)\(.*\n\)\2/&${message}\2/ + t exit + N + /printk(KERN_WARNING \"${file##*/}:/ b exit + s/\(.*\)\(\n${TAB}\)\(.*\n\)\2/&${message}\2/ + t exit + N + /printk(KERN_WARNING \"${file##*/}:/ b exit + s/\(.*\)\(\n${TAB}\)\(.*\n\)\2/&${message}\2/ + t exit + N + /printk(KERN_WARNING \"${file##*/}:/ b exit + s/\(.*\)\(\n${TAB}\)\(.*\n\)\2/&${message}\2/ + t exit + N + /printk(KERN_WARNING \"${file##*/}:/ b exit + s/\(.*\)\(\n${TAB}\)\(.*\n\)\2/&${message}\2/ + t exit + N + /printk(KERN_WARNING \"${file##*/}:/ b exit + s/\(.*\)\(\n${TAB}\)\(.*\n\)\2/&${message}\2/ + t exit + N + /printk(KERN_WARNING \"${file##*/}:/ b exit + s/\(.*\)\(\n${TAB}\)\(.*\n\)\2/&${message}\2/ + t exit + N + /printk(KERN_WARNING \"${file##*/}:/ b exit + s/\(.*\)\(\n${TAB}\)\(.*\n\)\2/&${message}\2/ + t exit + N + /printk(KERN_WARNING \"${file##*/}:/ b exit + s/\(.*\)\(\n${TAB}\)\(.*\n\)\2/&${message}\2/ + : exit + } + s/^builtin_\([a-z]*\)_driver(\(${symbol}\));\$/static int __init \2_init(void)\n{\n${TAB}${message}\n${TAB}return \1_driver_register(\&\2);\n}\ndevice_initcall(\2_init);/ + s/^module_\([a-z]*\)_driver(\(${symbol}\));\$/static int __init \2_init(void)\n{\n${TAB}${message}\n${TAB}return \1_driver_register(\&\2);\n}\nmodule_init(\2_init);\n\nstatic void __exit \2_exit(void)\n{\n${TAB}${message}\n${TAB}\1_driver_unregister(\&\2);\n}\nmodule_exit(\2_exit);/ + " ${file} > ${TMP}/${file##*/} && + ! cmp -s ${file} ${TMP}/${file##*/} && + cp ${TMP}/${file##*/} ${file} && + ( + echo INFO: modified ${file} to instrument ${call#select} ${symbol} >&2 + echo ${symbol} + ) || + ( + files=`grep -lr "^\(static \)\{0,1\}[^ ${TAB}][ %{TAB}_a-zA-Z0-9]* ${symbol}(" ${DIR} | + sed -e '/[.]h$/d' -e 's@^[.]/@@'` + if [ X"${file}" != X"${files}" ]; then + echo "${ORANGE}WARNING${NORMAL}: ${symbol} is in ${files}" >&2 + if [ 1 -eq `echo ${files} | wc -w` ]; then + file=${files} + add_debug_printk + exit + fi + fi + if [ "${SYMBOLS}" == "${SYMBOLS#* ${symbol}:${call#select}}" ]; then + echo "${RED}ERROR${NORMAL}: failed to modify ${file} to instrument" ${call#select} ${symbol} >&2 + fi + false + ) + ret=${?} + rm -f ${TMP}/${file##*/} + if [ 0 -eq ${ret} ]; then + SYMBOLS="${SYMBOLS}${symbol}:${call} " + fi + return ${ret} +} + +# module_init, module_exit, late_initcall ... +( + grep -Hr '^\(builtin_[a-z]*_driver\|module_[a-z]*_driver\|module_\(init\|exit\)\|[a-z]*_initcall\)(' ${DIR} | + sed -n 's@^[.]/@@ + s/^\([^:]*\):\(builtin_[a-z]*_driver\|module_[a-z]*_driver\|module_init\|module_exit\|[a-z]*_initcall\)(\([^)]*\));$/\1 \2 \3/p' | + sort -u + find ${DIR} -name '*.c' | + while read file; do + sed -n "/.* [a-z_0-9]*(void)\$/ { + : loop + N + /^[^\n]* [a-z_0-9]*(void)\n.*\n}\$/ { + /.*platform_\(driver\|device\)_\(un\)\{0,1\}register([&][^)]*).*/ { + s@^[^\n]* \([a-z_0-9]*\)(void)\n.*@${file} select \1@p + b exit + } + /.*[ ${TAB}]_[a-z]_\(un\)\{0,1\}register_driver([&][^, ${TAB})]*).*/ { + s@^[^\n]* \([a-z_0-9]*\)(void)\n.*@${file} select \1@p + b exit + } + /.*i2c_\(add\|del\)_driver([&][^)]*).*/ { + s@^[^\n]* \([a-z_0-9]*\)(void)\n.*@${file} select \1@p + } + b exit + } + b loop + } + : exit" ${file} + done | + sort -u +) | + while read file call symbol; do + if ! add_debug_printk && [ "${SYMBOLS}" == "${SYMBOLS#* ${symbol}:${call#select}}" ]; then + echo FAILED + fi + + if [ "${call}" != "${call#builtin_*_driver}" -o "${call}" != "${call#module_*_driver}" ]; then + symbol=${symbol}_init + fi + call=probe + sed -n "/\(static \)\{0,1\}.* ${symbol}(/,/^}/ { + s/.*platform_\(driver\|device\)_register([&]\([^)]*\)).*/struct platform_\1 \2/p + s/.*[ ${TAB}]\(_[a-z]\)_register_driver([&]\([^, ${TAB})]*\)).*/struct \1_driver \2/p + s/.*i2c_add_driver([&]\([^)]*\)).*/struct i2c_driver \1/p + } + s/\(builtin\|module\)_\([a-z]*_driver\)(\([^)]*\));/struct \2 \3/p" ${file} | + sort -u | + while read struct; do + sed -n "/\(static \)\{0,1\}${struct} = {/,/^};/ { + s/^[ ${TAB}]*[.]probe[ ${TAB}]*=[ ${TAB}]*\([^ ${TAB}]*\),\$/\1/p + }" ${file} | + sort -u | + while read symbol; do + if ! add_debug_printk && [ "${SYMBOLS}" == "${SYMBOLS#* ${symbol}:${call} }" ]; then + echo FAILED + fi + done + done + done | + sort -u | + tee ${TMP}/list | + grep '^FAILED' >/dev/null +ret=${?} +SYMBOLS=`grep -v '^FAILED$' ${TMP}/list` + +rm -rf ${TMP} + +if [ 0 -ne ${ret} ]; then + echo "DONE" >&2 + + AUTHOR_NAME="`git config user.name`" + AUTHOR_EMAIL="`git config user.email`" + if [ -z "${AUTHOR_EMAIL}" ]; then + AUTHOR_EMAIL="`${USER}@google.com`" + fi + if [ -z "${AUTHOR_NAME}" ]; then + convert_ldap_to_name() { + echo "${1##*/}" | + sed "s@[-_]@ @g + s@ *@ @g" | + sed 's@\(^\| \)a@\1A@g + s@\(^\| \)b@\1B@g + s@\(^\| \)c@\1C@g + s@\(^\| \)d@\1D@g + s@\(^\| \)e@\1E@g + s@\(^\| \)f@\1F@g + s@\(^\| \)g@\1G@g + s@\(^\| \)h@\1H@g + s@\(^\| \)i@\1I@g + s@\(^\| \)j@\1J@g + s@\(^\| \)k@\1K@g + s@\(^\| \)l@\1L@g + s@\(^\| \)m@\1M@g + s@\(^\| \)n@\1N@g + s@\(^\| \)o@\1O@g + s@\(^\| \)p@\1P@g + s@\(^\| \)q@\1Q@g + s@\(^\| \)r@\1R@g + s@\(^\| \)s@\1S@g + s@\(^\| \)t@\1T@g + s@\(^\| \)u@\1U@g + s@\(^\| \|Mc\)v@\1V@g + s@\(^\| \)w@\1W@g + s@\(^\| \)x@\1X@g + s@\(^\| \)y@\1Y@g + s@\(^\| \)z@\1Z@g' + } + AUTHOR_NAME="`convert_ldap_to_name ${USER}`" + fi + + git commit -a -m "GKI: DEBUG: add instrumentation to init and probe functions + +Automatically generated by ${0} + +The following symbols are instrumented: +`echo \"${SYMBOLS}\" | + sed 's/^/ - /'` + +Signed-off-by: ${AUTHOR_NAME} <${AUTHOR_EMAIL}> +Test: compile" || + echo "${RED}ERROR${NORMAL}: failed to commit changes to git" >&2 + exit +fi +echo "${RED}FAILED${NORMAL}: could not complete task" >&2 +exit 1 diff --git a/hermetic/Dockerfile b/hermetic/Dockerfile new file mode 100644 index 0000000000..70aa2f21d9 --- /dev/null +++ b/hermetic/Dockerfile @@ -0,0 +1,28 @@ +FROM debian:stable-slim + +RUN apt-get -y update \ +&& apt-get -y install \ +# Those are likely essential + git \ +# To be removed + rsync \ +&& apt-get clean \ +&& rm -rf /var/lib/apt/lists/* + +# Explicitly allow some host binaries for now +RUN mkdir -p /l/bin +RUN ln -s $(which perl) /l/bin/ +RUN ln -s $(which rsync) /l/bin/ +RUN ln -s $(which sh) /l/bin/ +RUN ln -s $(which tar) /l/bin/ # b/164013907 + +# Allow git usage during the build (e.g. to determine timestamps) +RUN ln -s $(which git) /l/bin/ + +# Copy the build runner +COPY build /l/build +RUN chmod a+rx /l/build + +# Run the build runner if no other command is given +CMD /l/build + diff --git a/hermetic/build b/hermetic/build new file mode 100755 index 0000000000..6443036780 --- /dev/null +++ b/hermetic/build @@ -0,0 +1,23 @@ +#!/bin/sh +# +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd /b/ + +# uncomment for incremental builds +#export SKIP_MRPROPER=1 + +export PATH=/l/bin:/b/build/build-tools/path/linux-x86 +BUILD_CONFIG=common/build.config.gki.aarch64 build/build.sh diff --git a/hermetic/run.sh b/hermetic/run.sh new file mode 100755 index 0000000000..a2c812c2d3 --- /dev/null +++ b/hermetic/run.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +BASE_DIR=$(readlink -f $(dirname $0)/../../) + +# Build the hermetic container +docker build -t hermetic $BASE_DIR/build/hermetic + +# Run the hermetic container +docker run -ti \ + --user $(id -u):$(id -g) \ + --mount type=bind,source=${BASE_DIR},target=/b/ \ + hermetic diff --git a/multi-switcher.sh b/multi-switcher.sh new file mode 100755 index 0000000000..cd73d65da2 --- /dev/null +++ b/multi-switcher.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +BASE=$(readlink -f $(dirname $0)/..) + +BRANCH=$1 + +pushd $BASE > /dev/null + + if [[ ! ( -d common-${BRANCH} || -d common${BRANCH} ) \ + || ${BRANCH} == "" \ + || ${BRANCH} == "modules" ]]; then + echo "usage: $0 " + echo + echo "Branches available: " + ls -d common-* | sed 's/common-/\t/g' | grep -v modules + ls -d common1?-* | sed 's/common/\t/g' + exit 1 + fi + + echo "Switching to $BRANCH" + + for dir in common common-modules/virtual-device; do + if [ -L ${dir} ]; then + rm ${dir} + fi + + for candidate in ${dir}-${BRANCH} ${dir}${BRANCH}; do + if [ -d ${candidate} ]; then + ( + cd $(dirname $candidate) + ln -vs $(basename ${candidate}) $(basename ${dir}) + ) + fi + done + done + +popd > /dev/null diff --git a/static_analysis/checkpatch.sh b/static_analysis/checkpatch.sh new file mode 100755 index 0000000000..cf229748ec --- /dev/null +++ b/static_analysis/checkpatch.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Wrapper around checkpatch.pl to filter results. + +set -e + +export STATIC_ANALYSIS_SRC_DIR=$(dirname $(readlink -f $0)) + +ROOT_DIR="${STATIC_ANALYSIS_SRC_DIR}/../../" +pushd ${ROOT_DIR} +source ${STATIC_ANALYSIS_SRC_DIR}/../_setup_env.sh +export OUT_DIR=$(readlink -m ${OUT_DIR:-${ROOT_DIR}/out/${BRANCH}}) +export DIST_DIR=$(readlink -m ${DIST_DIR:-${OUT_DIR}/dist}) +mkdir -p ${DIST_DIR} + +export KERNEL_DIR=$(readlink -m ${KERNEL_DIR}) + +CHECKPATCH_PL_PATH="${KERNEL_DIR}/scripts/checkpatch.pl" +GIT_SHA1="HEAD" +PATCH_DIR="${OUT_DIR}/checkpatch/patches" +IGNORELIST_FILE="${STATIC_ANALYSIS_SRC_DIR}/checkpatch_ignorelist" +RESULTS_PATH=${DIST_DIR}/checkpatch.log +RETURN_CODE=0 + +echoerr() { + echo "$@" 1>&2; +} + +# Parse flags. +CHECKPATCH_ARGS=(--show-types) +while [[ $# -gt 0 ]]; do + next="$1" + case ${next} in + --git_sha1) + GIT_SHA1="$2" + shift + ;; + --ignored_checks) + IGNORELIST_FILE="$2" + shift + ;; + --help) + echo "Gets a patch from git, passes it checkpatch.pl, and then reports" + echo "the subset of violations we choose to enforce." + echo "" + echo "Usage: $0" + echo " <--git_sha1 nnn> (Defaults to HEAD)" + echo " <--ignored_checks path_to_file> (Defaults to checkpatch_ignorelist)" + echo " " + exit 0 + ;; + *) + CHECKPATCH_ARGS+=("$1") + ;; + esac + shift +done + + +# Clean up from any previous run. +if [[ -d "${PATCH_DIR}" ]]; then + rm -fr "${PATCH_DIR}" +fi +mkdir -p "${PATCH_DIR}" + +# Update ignorelist. +if [[ -f "${IGNORELIST_FILE}" ]]; then + IGNORED_ERRORS=$(grep -v '^#' ${IGNORELIST_FILE} | paste -s -d,) + if [[ -n "${IGNORED_ERRORS}" ]]; then + CHECKPATCH_ARGS+=(--ignore) + CHECKPATCH_ARGS+=("${IGNORED_ERRORS}") + fi +fi + +echo "========================================================" +echo " Running static analysis..." +echo "========================================================" +echo "Using KERNEL_DIR: ${KERNEL_DIR}" +echo "Using --git_sha1: ${GIT_SHA1}" + +# Generate patch file from git. +cd ${KERNEL_DIR} +git format-patch --quiet -o "${PATCH_DIR}" "${GIT_SHA1}^1..${GIT_SHA1}" -- \ + ':!android/abi*' +PATCH_FILE="${PATCH_DIR}/*.patch" + +if ! `stat -t ${PATCH_FILE} >/dev/null 2>&1`; then + echo "Patch empty (probably due to suppressions). Skipping analysis." + exit 0 +fi + +# Delay exit on non-zero checkpatch.pl return code so we can finish logging. + +# Note, it's tricky to ignore this exit code completely and instead return only +# based on the log values. For example, if the log is not empty, but contains +# no ERRORS, how do we reliabliy distinguish WARNINGS that were not ignored (or +# other conditions we want to ignore), from legitimate errors running the +# script itself (e.g. bad flags)? checkpatch.pl will return 1 in both cases. +# For now, include all known warnings in the ignorelist, and forward this code +# unconditionally. + +set +e +"${CHECKPATCH_PL_PATH}" ${CHECKPATCH_ARGS[*]} $PATCH_FILE > "${RESULTS_PATH}" +CHECKPATCH_RC=$? +set -e + +# Summarize errors in the build log (full copy included in dist dir). +if [[ $CHECKPATCH_RC -ne 0 ]]; then + echoerr "Errors were reported from checkpatch.pl." + echoerr "" + echoerr "Summary:" + echoerr "" + { grep -r -h -E -A1 "^(ERROR|WARNING):" "${RESULTS_PATH}" 1>&2; } || true + echoerr "" + echoerr "See $(basename ${RESULTS_PATH}) for complete output." +fi + +echo "========================================================" +echo "Finished running static analysis." +echo "========================================================" +popd +exit ${CHECKPATCH_RC} diff --git a/static_analysis/checkpatch_ignorelist b/static_analysis/checkpatch_ignorelist new file mode 100644 index 0000000000..c2bf5f298c --- /dev/null +++ b/static_analysis/checkpatch_ignorelist @@ -0,0 +1,130 @@ +# Lists errors that should be ignored. +# To see the type of an error, run checkpatch with --show_types. +GERRIT_CHANGE_ID +# All WARNINGS are ignored +# TODO(mikewallstedt): Consider modifying checkpatch.pl such that we can +# ignore by level (this is non-trivial now because we have dozens of versions). +ALLOC_ARRAY_ARGS +ALLOC_WITH_MULTIPLY +ARM_BARRIER +ARRAY_SIZE +AVOID_EXTERNS +BAD_AUTHOR +BAD_SIGN_OFF +BLOCK_COMMENT_STYLE +BRACES +COMMIT_LOG_LONG_LINE +CONFIG_DESCRIPTION +CONFIG_EXPERIMENTAL +CONFIG_TYPE_BOOLEAN +CONSIDER_COMPLETION +CONSIDER_KSTRTO +CONSTANT_COMPARISON +CONSTANT_CONVERSION +CONST_CONST +CONST_STRUCT +CVS_KEYWORD +DEEP_INDENTATION +DEFAULT_NO_BREAK +DEFINE_PCI_DEVICE_TABLE +DEPRECATED_VARIABLE +DEPRECATED_VREG_APIS +DO_WHILE_MACRO_WITH_TRAILING_SEMICOLON +EMAIL_SUBJECT +ENOSYS +EXPEDITED_RCU_GRACE_PERIOD +EXPORTED_WORLD_WRITABLE +EXPORT_SYMBOL +FILE_PATH_CHANGES +FUNCTION_ARGUMENTS +GCC_BINARY_CONSTANT +GIT_COMMIT_ID +GPIOMUX_IN_BOARD +HEXADECIMAL_BOOLEAN_TEST +HOTPLUG_SECTION +IF_0 +IF_1 +IN_ATOMIC +INCLUDE_LINUX +INDENTED_LABEL +INLINE +JIFFIES_COMPARISON +KREALLOC_ARG_REUSE +LEADING_SPACE +LIKELY_MISUSE +LINE_CONTINUATIONS +LINE_SPACING +LINUX_VERSION_CODE +LONG_COMMIT_TEXT +LONG_SUMMARY_LINE +LONG_UDELAY +MACRO_WITH_FLOW_CONTROL +MASK_THEN_SHIFT +MDELAY +MEMORY_BARRIER +MEMSET +MINMAX +MISORDERED_TYPE +MISPLACED_INIT +MISSING_BREAK +MISSING_EOF_NEWLINE +MISSING_VMLINUX_SYMBOL +MODULE_LICENSE +MSLEEP +MULTIPLE_DECLARATION +NAKED_SSCANF +NETWORKING_BLOCK_COMMENT_STYLE +NEW_TYPEDEFS +NO_COMMIT_TEXT +NONBLANK_AFTER_SUMMARY +NR_CPUS +ONE_SEMICOLON +OOM_MESSAGE +PATCH_PREFIX +PREFER_ALIGNED +PREFER_DEV_LEVEL +PREFER_ETH_BROADCAST_ADDR +PREFER_ETHER_ADDR_COPY +PREFER_ETHER_ADDR_EQUAL +PREFER_ETH_ZERO_ADDR +PREFER_PACKED +PREFER_PRINTF +PREFER_PR_LEVEL +PREFER_SCANF +PREFER_SEQ_PUTS +PRINTF_L +PRINTK_RATELIMITED +PRINTK_WITHOUT_KERN_LEVEL +QUOTED_WHITESPACE_BEFORE_NEWLINE +RETURN_VOID +SINGLE_STATEMENT_DO_WHILE_MACRO +SIZEOF_ADDRESS +SIZEOF_PARENTHESIS +SPACE_BEFORE_TAB +SPACING +SPLIT_STRING +SSCANF_TO_KSTRTO +STATIC_CONST_CHAR_ARRAY +STORAGE_CLASS +STRING_FRAGMENTS +SUSPECT_CODE_INDENT +TRAILING_SEMICOLON +TYPO_SPELLING +UNDOCUMENTED_DT_STRING +UNNECESSARY_BREAK +UNNECESSARY_CASTS +UNNECESSARY_ELSE +UNNECESSARY_KERN_LEVEL +UNNECESSARY_PARENTHESES +USE_DEVICE_INITCALL +USE_FUNC +USE_NEGATIVE_ERRNO +USE_RELATIVE_PATH +USE_SPINLOCK_T +USLEEP_RANGE +UTF8_BEFORE_PATCH +VOLATILE +WAITQUEUE_ACTIVE +WHITESPACE_AFTER_LINE_CONTINUATION +YIELD + diff --git a/static_analysis/checkpatch_presubmit.sh b/static_analysis/checkpatch_presubmit.sh new file mode 100755 index 0000000000..e46723b3a0 --- /dev/null +++ b/static_analysis/checkpatch_presubmit.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Wrapper around checkpatch.sh to gather necessary information from the +# dist dir. Notably, this includes the git_sha1 and whether to suppress +# the check for post-submit. + +set -e + +export STATIC_ANALYSIS_SRC_DIR=$(dirname $(readlink -f $0)) + +source ${STATIC_ANALYSIS_SRC_DIR}/../_setup_env.sh +export OUT_DIR=$(readlink -m ${OUT_DIR:-${ROOT_DIR}/out/${BRANCH}}) +export DIST_DIR=$(readlink -m ${DIST_DIR:-${OUT_DIR}/dist}) + +APPLIED_PROP_PATH=${DIST_DIR}/applied.prop +BUILD_INFO_PATH=${DIST_DIR}/BUILD_INFO + +verify_file_exists() { + if [[ ! -f $1 ]]; then + echo "Missing $1" + exit 1 + fi +} + +# Parse flags. +BUILD_ID="" +FORWARDED_ARGS=() +while [[ $# -gt 0 ]]; do + next="$1" + case ${next} in + --bid) + BUILD_ID="$2" + shift + ;; + --help) + echo "Checks whether given build is for presubmit. If so, extract git_sha1" + echo "from repo.prop and invoke checkpatch.sh." + echo "" + echo "Usage: $0" + echo " <--bid nnn> (The build ID. Required.)" + echo " " + exit 0 + ;; + *) + FORWARDED_ARGS+=("$1") + ;; + esac + shift +done + +if [[ -z $BUILD_ID ]]; then + echo "WARNING: No --bid supplied. Assuming not presubmit build. Exiting." + exit 0 +fi + +# Skip checkpatch for postsubmit (b/35390488). +set +e +echo "${BUILD_ID}" | grep -E "^P[0-9]+" +if [[ $? -ne 0 ]]; then + echo "Did not identify a presubmit build. Exiting." + exit 0 +fi +set -e + +# Pick the correct patch to test. +verify_file_exists ${APPLIED_PROP_PATH} +GIT_SHA1=$(sed -nE "s#${KERNEL_DIR} .*([0-9a-f]{40}).*#\\1#p" "${APPLIED_PROP_PATH}") +if [[ -z ${GIT_SHA1} ]]; then + # Since applied.prop only tracks user changes, ignore projects that are + # included in presubmit without any changed files. + echo "No changes to apply for ${KERNEL_DIR}." + exit 0 +fi + +# Skip checkpatch for merge commits on common kernels +# These are upstream merges that likely hit issues in checkpatch.pl or merges +# from other common kernel repositories where _this_ check has been run as part +# of the developement process. +if [ "${KERNEL_DIR}" == "common" ]; then + if [ $(git -C ${KERNEL_DIR} show --no-patch --format="%p" ${GIT_SHA1} | wc -w) -gt 1 ] ; then + echo "Merge commit detected for a ${KERNEL_DIR} kernel. Skipping this check." + exit 0 + fi + + SUBJECT=$(git -C ${KERNEL_DIR} show --no-patch --format="%s" ${GIT_SHA1}) + if [[ "$SUBJECT" =~ ^UPSTREAM|^BACKPORT|^FROMGIT ]]; then + echo "Not linting upstream patches for a ${KERNEL_DIR} kernel. Skipping this check." + exit 0 + fi +fi + +${STATIC_ANALYSIS_SRC_DIR}/checkpatch.sh --git_sha1 ${GIT_SHA1} ${FORWARDED_ARGS[*]} diff --git a/synchronize_owners b/synchronize_owners new file mode 100755 index 0000000000..e96009d1a2 --- /dev/null +++ b/synchronize_owners @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2021 The Android Open main Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import argparse +import os +import pathlib +import sys + + +def process_owners(main_dir, main_branch, tracking_dirs): + # walk the tree ... + for subdir, _, files in os.walk(main_dir): + for file in files: + # ... and for each OWNERS file in the main ... + if file == 'OWNERS': + rel_dir = pathlib.Path(subdir[len(main_dir.name) + 1:]) + owners = rel_dir / file + + for tracking_dir in tracking_dirs: + target = tracking_dir / owners + comment_line = (f'# include OWNERS from the authoritative ' + f'{main_branch} branch\n') + include_line = f'include kernel/common:{main_branch}:/{owners}\n' + + # ... either add a new file with include directive ... + if not target.is_file(): + with open(target, 'w') as file: + file.writelines([comment_line, include_line]) + + # ... or ensure the include directive is in the file + else: + with open(target) as file: + contents = file.readlines() + if include_line not in contents: + with open(target, 'w') as file: + file.writelines([comment_line, include_line, '\n'] + contents) + + +def is_dir(dirname): + result = pathlib.Path(dirname) + if not result.is_dir(): + raise argparse.ArgumentTypeError(f'{dirname} is not a directory') + else: + return result + + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument( + 'main_dir', type=is_dir, help='main directory to synchronize from') + parser.add_argument('main_branch', help='main branch name'), + parser.add_argument( + 'tracking_dir', type=is_dir, nargs='+', help='tracking directory to synchronize to') + + args = parser.parse_args() + + process_owners(args.main_dir, args.main_branch, args.tracking_dir) + + +if __name__ == '__main__': + sys.exit(main())